@frontera-sdk/cli 1.52.39 → 1.52.40

package/src/local/local-sample-data.ts

@@ -0,0 +1,141 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { SCAFFOLD_SAMPLE_ROWS } from './local-scaffold'
+
+/**
+ * The rule set `frontera local data --synthetic` writes sample rows by,
+ * stamped into each dataset it writes (in its description, the only free
+ * text a dataset definition takes). Raise it whenever a change to the
+ * generator makes rows written before it break a rule it now keeps — a key
+ * that is another type's key taking each record once, a share staying
+ * between 0 and 100, times of day as HH:MM, a cancelled record carrying no
+ * resolution — so a project whose sample data is older is told to remake it.
+ *
+ *   1  (unstamped) everything before the stamp
+ *   2  1:1 key links, shares 0–100, HH:MM times, state-dependent fields, descriptions by kind
+ *   3  a link to a media-backed type's file holds a placeholder's path, so it resolves
+ *   4  a property without `required` is optional (state rules apply to it); an id named
+ *      for another type takes its keys; counts, periods, fiscal years and keys are plausible
+ *   5  a property implementing a shared field takes the value of the linked record that
+ *      implements it too (a work order's service region is its site's)
+ *   6  a field conditional on something its record has nothing for ("empty unless
+ *      escalated", no `escalated` property or value) stays empty instead of random
+ *   7  a number keeps to the range its description gives ("(1 to 12)", "between 1 and 5"),
+ *      a "How many" count stays up to 10, and a city is the large city a custom point is near
+ */
+export const SAMPLE_DATA_RULES = 7
+
+const COMMAND = '`frontera local data --synthetic`'
+const MADE_BY = /made by `frontera local data --synthetic`(?: \(sample rules (\d+)\))?/
+
+/** The words a sample dataset's description carries: who made it, by which rules. */
+export function sampleStamp(rules = SAMPLE_DATA_RULES): string {
+  return `made by ${COMMAND} (sample rules ${rules})`
+}
+
+/** The description of a sample dataset, with the current stamp: the one it had, restamped, or a new one. */
+export function stampedDescription(before: string | undefined, fresh: string): string {
+  if (!before) return fresh
+  if (MADE_BY.test(before)) return before.replace(MADE_BY, sampleStamp())
+  // A description of its own, on a dataset whose file had no rows yet: kept, and said to hold sample rows.
+  return `${before.replace(/\s+$/, '')} Its rows are sample records ${sampleStamp()}: not real records.`
+}
+
+/** The rule set a dataset's description says its rows were made by: 1 for an unstamped sample, null when not sample data. */
+export function sampleRulesOf(description: unknown): number | null {
+  if (typeof description !== 'string') return null
+  const m = MADE_BY.exec(description)
+  return m ? (m[1] ? Number(m[1]) : 1) : null
+}
+
+/**
+ * Whether `frontera local data --synthetic` may write a dataset's rows: a
+ * dataset it made (its description says so), one with no rows yet, or the
+ * scaffold's sample rows unedited. Anything else — a dataset someone wrote,
+ * edited, loaded from a Source or brought in — is kept as it is.
+ */
+export function isSampleDataset(root: string, dataset: string): boolean {
+  const definition = readJson(join(root, 'datasets', `${dataset}.json`))
+  const csvPath = join(root, 'datasets', `${dataset}.csv`)
+  if (!definition) return !existsSync(csvPath) || !hasRows(readFileSync(csvPath, 'utf8'))
+  if (sampleRulesOf(definition.description) !== null) return true
+  if (definition.kind !== undefined && definition.kind !== 'file') return false
+  const rel = typeof definition.path === 'string' ? definition.path.replace(/^\.\//, '') : `${dataset}.csv`
+  const csv = join(root, 'datasets', rel)
+  if (!existsSync(csv)) return true
+  const text = readFileSync(csv, 'utf8')
+  // Empty, or a header and nothing under it: no rows yet.
+  if (!hasRows(text)) return true
+  const scaffold = SCAFFOLD_SAMPLE_ROWS[dataset]
+  return scaffold !== undefined && text.replace(/\r\n/g, '\n') === scaffold
+}
+
+/**
+ * The sample datasets made by an older rule set than today's: their rows can
+ * break rules the generator keeps now. Each by its file, oldest rules first.
+ */
+export function staleSampleData(root: string, written?: ReadonlySet<string>): Array<{ file: string; rules: number }> {
+  const dir = join(root, 'datasets')
+  if (!existsSync(dir)) return []
+  return readdirSync(dir)
+    // Only the datasets the generator writes (`sampleDatasetNames`): a media-backed type's is never remade.
+    .filter((f) => f.endsWith('.json') && (!written || written.has(f.slice(0, -'.json'.length))))
+    .flatMap((f) => {
+      const rules = sampleRulesOf(readJson(join(dir, f))?.description)
+      return rules !== null && rules < SAMPLE_DATA_RULES ? [{ file: `datasets/${f}`, rules }] : []
+    })
+    .sort((a, b) => a.rules - b.rules || a.file.localeCompare(b.file))
+}
+
+/** What `local dev` and `local status` say about stale sample data, or null when there is none. */
+export function staleSampleDataLine(root: string, written?: ReadonlySet<string>): string | null {
+  const stale = staleSampleData(root, written)
+  if (!stale.length) return null
+  const names = stale.slice(0, 4).map((s) => s.file.replace(/^datasets\/|\.json$/g, '')).join(', ')
+  return `sample data made by an older generator (${stale.length} dataset${stale.length === 1 ? '' : 's'}: ${names}${stale.length > 4 ? ', …' : ''}): its rows can break rules the generator keeps now (links by a shared key or by an id's name, shares 0–100, HH:MM times, links to media files, fields empty by the record's state); \`frontera local dev --remake-sample-data\` or the next \`frontera local sync pull\` makes them again with the kept seed, rows and day, or ${COMMAND} remakes the sample datasets and leaves every other data file as it is`
+}
+
+/** Whether a CSV holds a record under its header: an empty file or a header alone holds none. */
+function hasRows(text: string): boolean {
+  return readCsvRows(text).length > 0
+}
+
+function readJson(path: string): Record<string, unknown> | null {
+  try {
+    const doc = JSON.parse(readFileSync(path, 'utf8')) as unknown
+    return doc && typeof doc === 'object' && !Array.isArray(doc) ? (doc as Record<string, unknown>) : null
+  } catch {
+    return null
+  }
+}
+
+/** A CSV's rows by header (RFC 4180 quoting), for a dataset sample data keeps and links into. */
+export function readCsvRows(text: string): Array<Record<string, string>> {
+  const records: string[][] = []
+  let field = ''
+  let record: string[] = []
+  let quoted = false
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i]!
+    if (quoted) {
+      if (ch === '"' && text[i + 1] === '"') { field += '"'; i++ }
+      else if (ch === '"') quoted = false
+      else field += ch
+      continue
+    }
+    if (ch === '"') quoted = true
+    else if (ch === ',') { record.push(field); field = '' }
+    else if (ch === '\n' || ch === '\r') {
+      if (ch === '\r' && text[i + 1] === '\n') i++
+      record.push(field); field = ''
+      if (record.some((v) => v !== '')) records.push(record)
+      record = []
+    } else field += ch
+  }
+  record.push(field)
+  if (record.some((v) => v !== '')) records.push(record)
+  const [header, ...rows] = records
+  if (!header) return []
+  return rows.map((r) => Object.fromEntries(header.map((h, i) => [h, r[i] ?? ''])))
+}

package/src/local/local-sample-remake.ts

@@ -0,0 +1,138 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { dirname, join } from 'node:path'
+
+import { SAMPLE_DATA_RULES, sampleRulesOf, staleSampleData } from './local-sample-data'
+import { readLock } from './local-sync-plan'
+import { generateSyntheticData, loadScenario, parseRegion, sampleDatasetNames, type GeneratedDataset } from './local-synthetic-data'
+
+/**
+ * Sample data made again on its own: after a pull that brought Blueprint
+ * items (a type's properties or links may have changed), or when the sample
+ * datasets were made by an older generator (`staleSampleData`). Every sample
+ * dataset is made again together, so their links agree; only the datasets
+ * `isSampleDataset` lets the generator write (stamped by it, scaffold rows
+ * unedited, or empty), never a dataset someone wrote, edited, loaded or
+ * pulled. A project that has no generated sample data is left alone, and so
+ * are the Blueprint files (no backing is added, as `local data` adds one).
+ *
+ * The generator's options are kept in `.frontera/sample-data.json`, so a
+ * remake writes the same keys and the same rows as the last run did.
+ */
+
+export const SAMPLE_OPTIONS_FILE = join('.frontera', 'sample-data.json')
+
+export interface SampleOptions {
+  seed: number
+  rows: number
+  /** The day due dates are spread around, YYYY-MM-DD. */
+  today: string
+  /** `--region` as it was given, when it was. */
+  region?: string
+  /**
+   * What last made the sample data: `pull` (a `local sync pull`, or `local
+   * init --from-workspace`, wrote it with the pulled files) or `local` (`local
+   * data --synthetic`, or `local dev` making it again). Absent in a record
+   * from an older CLI.
+   */
+  madeBy?: 'pull' | 'local'
+}
+
+const DAY = /^\d{4}-\d{2}-\d{2}$/
+
+export const isoDay = (at: Date) => at.toISOString().slice(0, 10)
+
+export function readSampleOptions(root: string): SampleOptions | null {
+  try {
+    const doc = JSON.parse(readFileSync(join(root, SAMPLE_OPTIONS_FILE), 'utf8')) as Partial<SampleOptions>
+    if (!Number.isInteger(doc.seed) || !Number.isInteger(doc.rows) || typeof doc.today !== 'string' || !DAY.test(doc.today)) return null
+    return { seed: doc.seed!, rows: doc.rows!, today: doc.today, ...(typeof doc.region === 'string' && doc.region ? { region: doc.region } : {}), ...(doc.madeBy === 'pull' || doc.madeBy === 'local' ? { madeBy: doc.madeBy } : {}) }
+  } catch {
+    return null
+  }
+}
+
+export function writeSampleOptions(root: string, options: SampleOptions): void {
+  const path = join(root, SAMPLE_OPTIONS_FILE)
+  mkdirSync(dirname(path), { recursive: true })
+  writeFileSync(path, `${JSON.stringify(options, null, 2)}\n`)
+}
+
+/** Whether the project holds sample data the generator made (a stamped dataset), so it has some to remake. */
+export function hasGeneratedSampleData(root: string, names: ReadonlySet<string> = sampleDatasetNames(root)): boolean {
+  for (const name of names) {
+    const path = join(root, 'datasets', `${name}.json`)
+    if (!existsSync(path)) continue
+    try {
+      if (sampleRulesOf((JSON.parse(readFileSync(path, 'utf8')) as { description?: unknown }).description) !== null) return true
+    } catch {
+      // Not a definition the generator wrote.
+    }
+  }
+  return false
+}
+
+export interface SampleRemake {
+  /** Why it was made again, in words. */
+  why: string
+  options: SampleOptions
+  written: GeneratedDataset[]
+  kept: GeneratedDataset[]
+}
+
+/**
+ * Make the project's sample data again with its kept options (seed 7, 200
+ * rows and today when none are kept, which are then kept). Null when the
+ * project has no generated sample data.
+ */
+export async function remakeSampleData(root: string, why: string, now: Date = new Date(), madeBy: 'pull' | 'local' = 'local'): Promise<SampleRemake | null> {
+  const names = sampleDatasetNames(root)
+  if (!names.size || !hasGeneratedSampleData(root, names)) return null
+  const { madeBy: _before, ...kept } = readSampleOptions(root) ?? { seed: 7, rows: 200, today: isoDay(now) }
+  const options: SampleOptions = { ...kept, madeBy }
+  const region = options.region ? parseRegion(options.region) ?? undefined : undefined
+  const scenario = await loadScenario(root, options.seed)
+  const made = generateSyntheticData(root, { rows: options.rows, seed: options.seed, today: new Date(`${options.today}T00:00:00Z`), ...(region ? { region } : {}), ...(scenario ? { scenario } : {}) })
+  writeSampleOptions(root, options)
+  return { why, options, written: made.filter((m) => !m.kept), kept: made.filter((m) => m.kept) }
+}
+
+/**
+ * Why the project's sample data should be made again now, or null: the pull
+ * brought Blueprint items, or an older generator made it.
+ */
+export function remakeReason(root: string, pulledBlueprint: boolean): string | null {
+  const stale = staleSampleData(root, sampleDatasetNames(root))
+  if (stale.length) return `made by an older generator (sample rules ${stale[0]!.rules})`
+  if (pulledBlueprint) return 'the pull brought Blueprint items'
+  return null
+}
+
+/**
+ * Whether a pulled project's sample data is the pull's own: the project was
+ * pulled (`frontera.lock`) and nothing recorded says the local generator made
+ * it since (`madeBy: local` in .frontera/sample-data.json, which `local data
+ * --synthetic` and a remake by `local dev` write). A record from an older CLI
+ * says neither, and counts as the pull's: the data predates this CLI's runs.
+ * `local dev` doesn't make such data again unasked; `local dev
+ * --remake-sample-data`, a pull, or `local data --synthetic` does.
+ */
+export function pulledSampleData(root: string): boolean {
+  return readLock(root) !== null && readSampleOptions(root)?.madeBy !== 'local'
+}
+
+/** What `local dev` says instead of making the pull's sample data again: one line, with the command that does. */
+export function pulledSampleDataNote(root: string): string {
+  const stale = staleSampleData(root, sampleDatasetNames(root))
+  const n = new Set(stale.map((s) => s.file)).size
+  return `Sample data: newer sample rules exist (${SAMPLE_DATA_RULES}; ${n} dataset${n === 1 ? '' : 's'} came with the pull at rules ${stale[0]?.rules ?? 'older'}), kept as pulled; \`frontera local dev --remake-sample-data\` makes it again with the kept seed, rows and day`
+}
+
+/** The lines a command says of a remake. */
+export function remakeLines(r: SampleRemake): string[] {
+  const rows = r.written.reduce((n, d) => n + d.rows, 0)
+  return [
+    `Sample data made again (${r.why}): ${rows} rows for ${r.written.length} object type${r.written.length === 1 ? '' : 's'}, seed ${r.options.seed}, ${r.options.rows} rows a type, dates around ${r.options.today}${r.options.region ? `, region ${r.options.region}` : ''} (kept in ${SAMPLE_OPTIONS_FILE}); --keep-sample-data leaves it as it was`,
+    ...r.written.map((d) => `  ${d.objectType.padEnd(24)} ${String(d.rows).padStart(6)}  ${d.csvPath}`),
+    ...(r.kept.length ? [`  kept as they are (not sample data): ${r.kept.map((d) => d.csvPath).join(', ')}`] : []),
+  ]
+}