@auto-engineer/narrative 0.11.11

package/src/getNarratives.ts

@@ -0,0 +1,186 @@
+import createDebug from 'debug';
+import { registry } from './narrative-registry';
+import type { Narrative, Model } from './index';
+import { narrativesToModel } from './transformers/narrative-to-model';
+import type { IFileStore } from '@auto-engineer/file-store';
+import { executeAST } from './loader';
+import { sha256 } from 'js-sha256';
+
+const dirnamePosix = (p: string) => {
+  const s = p.replace(/\/+$/, '');
+  const i = s.lastIndexOf('/');
+  return i > 0 ? s.slice(0, i) : '/';
+};
+
+const debug = createDebug('auto:narrative:getNarratives');
+
+const toPosix = (p: string) => p.replace(/\\/g, '/');
+const DEFAULT_PATTERN = /\.(narrative|integration)\.(ts|tsx|js|jsx|mjs|cjs)$/;
+const DEFAULT_IGNORE_DIRS = /(?:^|\/)(?:node_modules|dist|\.turbo|\.git)(?:\/|$)/;
+const DTS_PATTERN = /\.d\.ts$/;
+const CACHE_VERSION = 'v1';
+
+function stableStringify(obj: Record<string, unknown>) {
+  const keys = Object.keys(obj).sort();
+  return JSON.stringify(keys.reduce((a, k) => ((a[k] = obj[k]), a), {} as Record<string, unknown>));
+}
+
+function toUint8(view: ArrayBuffer | Uint8Array): Uint8Array {
+  return view instanceof Uint8Array ? view : new Uint8Array(view);
+}
+
+async function hashFiles(vfs: IFileStore, files: string[]): Promise<string> {
+  const sorted = [...new Set(files)].sort();
+  const hasher = sha256.create();
+  const enc = new TextEncoder();
+
+  for (const f of sorted) {
+    hasher.update(enc.encode(f));
+    const buf = await vfs.read(f);
+    if (buf) hasher.update(toUint8(buf));
+  }
+
+  return hasher.hex();
+}
+
+export interface GetNarrativesOptions {
+  vfs: IFileStore;
+  root: string;
+  pattern?: RegExp;
+  importMap?: Record<string, unknown>;
+  fastFsScan?: boolean;
+}
+
+interface CacheEntry {
+  result: {
+    narratives: Narrative[];
+    vfsFiles: string[];
+    externals: string[];
+    typings: Record<string, string[]>;
+    typeMap: Map<string, string>;
+    typesByFile: Map<string, Map<string, unknown>>;
+    givenTypesByFile: Map<string, unknown[]>;
+    toModel: () => Model;
+  };
+  contentHash: string;
+}
+
+const compilationCache = new Map<string, CacheEntry>();
+
+async function discoverFiles(vfs: IFileStore, root: string, pattern: RegExp, fastScan: boolean): Promise<string[]> {
+  const entries = await vfs.listTree(root, {
+    includeSizes: false,
+    followSymlinkDirs: !fastScan,
+    pruneDirRegex: DEFAULT_IGNORE_DIRS,
+  });
+  const files = entries
+    .filter((e) => e.type === 'file')
+    .map((e) => toPosix(e.path))
+    .filter((p) => !DEFAULT_IGNORE_DIRS.test(p))
+    .filter((p) => !DTS_PATTERN.test(p))
+    .filter((p) => pattern.test(p))
+    .sort();
+
+  debug('discover: root=%s pattern=%s matched=%d', root, String(pattern), files.length);
+  return files;
+}
+
+async function validateCache(
+  cached: CacheEntry,
+  seedFiles: string[],
+  vfs: IFileStore,
+): Promise<CacheEntry['result'] | null> {
+  const prevGraphFiles = cached.result.vfsFiles;
+  const seedSet = new Set(seedFiles);
+  const prevSet = new Set(prevGraphFiles);
+
+  const hasNewFiles = seedFiles.some((f) => !prevSet.has(f));
+  const hasRemovedFiles = prevGraphFiles.some((f) => !seedSet.has(f));
+
+  if (hasNewFiles || hasRemovedFiles) {
+    debug('cache invalidated (file list changed: +%d -%d)', hasNewFiles ? 1 : 0, hasRemovedFiles ? 1 : 0);
+    return null;
+  }
+
+  const prevHash = await hashFiles(vfs, prevGraphFiles);
+  if (prevHash === cached.contentHash) {
+    debug('cache hit (graph unchanged)');
+    return cached.result;
+  }
+  debug('cache invalidated (graph changed)');
+  return null;
+}
+
+export const getNarratives = async (
+  opts: GetNarrativesOptions,
+): Promise<{
+  narratives: Narrative[];
+  vfsFiles: string[];
+  externals: string[];
+  typings: Record<string, string[]>;
+  typeMap: Map<string, string>;
+  typesByFile: Map<string, Map<string, unknown>>;
+  givenTypesByFile: Map<string, unknown[]>;
+  toModel: () => Model;
+}> => {
+  const { vfs, root, pattern = DEFAULT_PATTERN, importMap = {} } = opts;
+  const normRoot = toPosix(root);
+  const projectRoot = dirnamePosix(normRoot);
+
+  const seedFiles = await discoverFiles(
+    vfs,
+    normRoot,
+    pattern,
+    opts.fastFsScan === undefined ? false : opts.fastFsScan,
+  );
+  if (seedFiles.length === 0) {
+    debug('no candidate files found. root=%s pattern=%s', normRoot, String(pattern));
+    return {
+      narratives: [],
+      vfsFiles: [],
+      externals: [],
+      typings: {},
+      typeMap: new Map(),
+      typesByFile: new Map(),
+      givenTypesByFile: new Map(),
+      toModel: () => ({ variant: 'specs' as const, narratives: [], messages: [] }),
+    };
+  }
+
+  const cacheKey = [CACHE_VERSION, normRoot, String(pattern), stableStringify(importMap)].join('|');
+  const cached = compilationCache.get(cacheKey);
+
+  if (cached) {
+    const cachedResult = await validateCache(cached, seedFiles, vfs);
+    if (cachedResult) {
+      return cachedResult;
+    }
+  }
+  registry.clearAll();
+
+  const exec = await executeAST(seedFiles, vfs, importMap, projectRoot);
+
+  const narratives: Narrative[] = registry.getAllNarratives();
+  const result = {
+    narratives,
+    vfsFiles: exec.vfsFiles, // absolute posix paths of all VFS modules in the graph
+    externals: exec.externals, // external specifiers used
+    typings: exec.typings, // { pkgName: [abs d.ts paths] }
+    typeMap: exec.typeMap, // mapping from TypeScript type names to string literals
+    typesByFile: exec.typesByFile, // mapping from file path to type definitions in that file
+    givenTypesByFile: exec.givenTypesByFile, // mapping from file path to given type info
+    toModel: (): Model => narrativesToModel(narratives, exec.typesByFile),
+  };
+
+  const contentHash = await hashFiles(vfs, exec.vfsFiles);
+
+  compilationCache.set(cacheKey, { result, contentHash });
+  debug('cached compilation result, hash=%s', contentHash.slice(0, 8));
+
+  return result;
+};
+
+export const clearGetNarrativesCache = (): void => {
+  compilationCache.clear();
+  debug('cleared compilation cache');
+};

package/src/id/addAutoIds.specs.ts

@@ -0,0 +1,296 @@
+import { describe, it, expect } from 'vitest';
+import { addAutoIds } from './';
+import { Model } from '../index';
+
+describe('addAutoIds', () => {
+  const flows: Model = {
+    variant: 'specs',
+    narratives: [
+      {
+        name: 'Test Flow',
+        slices: [
+          {
+            type: 'command',
+            name: 'Test Command Slice',
+            client: { description: 'Test client' },
+            server: {
+              description: 'Test server',
+              specs: {
+                name: 'Test Specs',
+                rules: [
+                  {
+                    description: 'Test rule without ID',
+                    examples: [],
+                  },
+                  {
+                    id: 'EXISTING-RULE-001',
+                    description: 'Test rule with existing ID',
+                    examples: [],
+                  },
+                ],
+              },
+            },
+          },
+          {
+            type: 'query',
+            name: 'Test Query Slice',
+            id: 'EXISTING-SLICE-001',
+            client: { description: 'Test client' },
+            server: {
+              description: 'Test server',
+              specs: {
+                name: 'Test Specs',
+                rules: [],
+              },
+            },
+          },
+        ],
+      },
+      {
+        name: 'Flow with ID',
+        id: 'EXISTING-FLOW-001',
+        slices: [
+          {
+            type: 'react',
+            name: 'React Slice',
+            server: {
+              description: 'React server',
+              specs: {
+                name: 'React Specs',
+                rules: [
+                  {
+                    description: 'React rule',
+                    examples: [],
+                  },
+                ],
+              },
+            },
+          },
+        ],
+      },
+    ],
+    messages: [],
+    integrations: [],
+  };
+
+  const AUTO_ID_REGEX = /^AUTO-[A-Za-z0-9_]{9}$/;
+
+  it('should assign IDs to entities that do not have them', () => {
+    const result = addAutoIds(flows);
+
+    expect(result.narratives[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[1].id).toBe('EXISTING-FLOW-001');
+    expect(result.narratives[0].slices[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[0].slices[1].id).toBe('EXISTING-SLICE-001');
+    expect(result.narratives[1].slices[0].id).toMatch(AUTO_ID_REGEX);
+    const slice0 = result.narratives[0].slices[0];
+    const slice1 = result.narratives[1].slices[0];
+
+    if ('server' in slice0 && slice0.server?.specs?.rules != null) {
+      expect(slice0.server.specs.rules[0].id).toMatch(AUTO_ID_REGEX);
+      expect(slice0.server.specs.rules[1].id).toBe('EXISTING-RULE-001');
+    }
+
+    if ('server' in slice1 && slice1.server?.specs?.rules != null) {
+      expect(slice1.server.specs.rules[0].id).toMatch(AUTO_ID_REGEX);
+    }
+  });
+
+  it('should not mutate the original flows', () => {
+    const originalFlow = flows.narratives[0];
+    const originalSlice = originalFlow.slices[0];
+
+    addAutoIds(flows);
+
+    expect(originalFlow.id).toBeUndefined();
+    expect(originalSlice.id).toBeUndefined();
+    if (
+      'server' in originalSlice &&
+      originalSlice.server?.specs?.rules !== undefined &&
+      originalSlice.server.specs.rules.length > 0
+    ) {
+      expect(originalSlice.server.specs.rules[0].id).toBeUndefined();
+    }
+  });
+
+  it('should preserve existing IDs and not overwrite them', () => {
+    const result = addAutoIds(flows);
+
+    expect(result.narratives[1].id).toBe('EXISTING-FLOW-001');
+    expect(result.narratives[0].slices[1].id).toBe('EXISTING-SLICE-001');
+
+    const testSlice = result.narratives[0].slices[0];
+    if ('server' in testSlice && testSlice.server?.specs?.rules != null) {
+      expect(testSlice.server.specs.rules[1].id).toBe('EXISTING-RULE-001');
+    }
+  });
+
+  it('should handle flows without server blocks', () => {
+    const modelWithoutServer: Model = {
+      variant: 'specs',
+      narratives: [
+        {
+          name: 'Simple Flow',
+          slices: [
+            {
+              type: 'command',
+              name: 'Simple Command',
+              client: { description: 'Simple client' },
+              server: {
+                description: 'Simple server',
+                specs: {
+                  name: 'Simple specs',
+                  rules: [],
+                },
+              },
+            },
+          ],
+        },
+      ],
+      messages: [],
+      integrations: [],
+    };
+
+    const result = addAutoIds(modelWithoutServer);
+
+    expect(result.narratives[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[0].slices[0].id).toMatch(AUTO_ID_REGEX);
+  });
+
+  it('should generate unique IDs for multiple calls', () => {
+    const result1 = addAutoIds(flows);
+    const result2 = addAutoIds(flows);
+
+    expect(result1.narratives[0].id).not.toBe(result2.narratives[0].id);
+    expect(result1.narratives[0].slices[0].id).not.toBe(result2.narratives[0].slices[0].id);
+  });
+
+  it('should assign IDs to experience slices', () => {
+    const modelWithExperienceSlice: Model = {
+      variant: 'specs',
+      narratives: [
+        {
+          name: 'Experience Flow',
+          slices: [
+            {
+              type: 'experience',
+              name: 'User Onboarding Experience',
+              client: {
+                description: 'User onboarding client',
+                specs: {
+                  name: 'Onboarding Specs',
+                  rules: ['User should see welcome message', 'User should complete profile setup'],
+                },
+              },
+            },
+            {
+              type: 'experience',
+              name: 'Checkout Experience',
+              id: 'EXISTING-EXPERIENCE-SLICE-001',
+              client: {
+                description: 'Checkout client',
+                specs: {
+                  name: 'Checkout Specs',
+                  rules: ['User should review cart items'],
+                },
+              },
+            },
+          ],
+        },
+      ],
+      messages: [],
+      integrations: [],
+    };
+
+    const result = addAutoIds(modelWithExperienceSlice);
+
+    // Flow should get an auto ID
+    expect(result.narratives[0].id).toMatch(AUTO_ID_REGEX);
+
+    // Experience slices should get auto IDs where missing
+    expect(result.narratives[0].slices[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[0].slices[1].id).toBe('EXISTING-EXPERIENCE-SLICE-001');
+
+    // Experience slices only have client specs (no server specs to test)
+  });
+
+  it('should assign unique IDs to multiple flows with same sourceFile', () => {
+    const modelWithMultipleFlowsSameSource: Model = {
+      variant: 'specs',
+      narratives: [
+        {
+          name: 'Home Screen',
+          sourceFile: '/path/to/homepage.flow.ts',
+          slices: [
+            {
+              name: 'Active Surveys Summary',
+              type: 'experience',
+              client: {
+                specs: {
+                  name: '',
+                  rules: ['show active surveys summary'],
+                },
+              },
+            },
+          ],
+        },
+        {
+          name: 'Create Survey',
+          sourceFile: '/path/to/homepage.flow.ts',
+          slices: [
+            {
+              name: 'Create Survey Form',
+              type: 'experience',
+              client: {
+                specs: {
+                  name: '',
+                  rules: ['allow entering survey title'],
+                },
+              },
+            },
+          ],
+        },
+        {
+          name: 'Response Analytics',
+          sourceFile: '/path/to/homepage.flow.ts',
+          slices: [
+            {
+              name: 'Response Rate Charts',
+              type: 'experience',
+              client: {
+                specs: {
+                  name: '',
+                  rules: ['show daily response rate charts'],
+                },
+              },
+            },
+          ],
+        },
+      ],
+      messages: [],
+      integrations: [],
+    };
+
+    const result = addAutoIds(modelWithMultipleFlowsSameSource);
+
+    expect(result.narratives[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[1].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[2].id).toMatch(AUTO_ID_REGEX);
+
+    expect(result.narratives[0].id).not.toBe(result.narratives[1].id);
+    expect(result.narratives[0].id).not.toBe(result.narratives[2].id);
+    expect(result.narratives[1].id).not.toBe(result.narratives[2].id);
+
+    expect(result.narratives[0].slices[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[1].slices[0].id).toMatch(AUTO_ID_REGEX);
+    expect(result.narratives[2].slices[0].id).toMatch(AUTO_ID_REGEX);
+
+    expect(result.narratives[0].slices[0].id).not.toBe(result.narratives[1].slices[0].id);
+    expect(result.narratives[0].slices[0].id).not.toBe(result.narratives[2].slices[0].id);
+    expect(result.narratives[1].slices[0].id).not.toBe(result.narratives[2].slices[0].id);
+
+    expect(result.narratives[0].sourceFile).toBe('/path/to/homepage.flow.ts');
+    expect(result.narratives[1].sourceFile).toBe('/path/to/homepage.flow.ts');
+    expect(result.narratives[2].sourceFile).toBe('/path/to/homepage.flow.ts');
+  });
+});

package/src/id/addAutoIds.ts

@@ -0,0 +1,53 @@
+import { generateAutoId } from './generators';
+import { Model, Slice } from '../index';
+
+function ensureId(item: { id?: string }): void {
+  if (item.id === undefined || item.id === '') {
+    item.id = generateAutoId();
+  }
+}
+
+function addRuleIds(rules: Array<unknown>): Array<unknown> {
+  return rules.map((rule) => {
+    if (typeof rule === 'object' && rule !== null && 'description' in rule) {
+      const ruleCopy = { ...rule } as { id?: string };
+      ensureId(ruleCopy);
+      return ruleCopy;
+    }
+    return rule;
+  });
+}
+
+function processServerSpecs(slice: Slice): Slice {
+  if (!('server' in slice) || slice.server?.specs?.rules === undefined) return slice;
+
+  const modifiedSlice = structuredClone(slice);
+  if ('server' in modifiedSlice && modifiedSlice.server?.specs?.rules !== undefined) {
+    (modifiedSlice.server.specs as { rules: unknown[] }).rules = addRuleIds(modifiedSlice.server.specs.rules);
+  }
+  return modifiedSlice;
+}
+
+function processClientSpecs(slice: Slice): Slice {
+  // Client specs use string rules (no IDs needed), so nothing to process
+  return slice;
+}
+
+function processSlice(slice: Slice): Slice {
+  let sliceCopy = { ...slice };
+  ensureId(sliceCopy);
+  sliceCopy = processServerSpecs(sliceCopy);
+  sliceCopy = processClientSpecs(sliceCopy);
+  return sliceCopy;
+}
+
+export function addAutoIds(specs: Model): Model {
+  const result = structuredClone(specs);
+  result.narratives = result.narratives.map((narrative) => {
+    const narrativeCopy = { ...narrative };
+    ensureId(narrativeCopy);
+    narrativeCopy.slices = narrative.slices.map(processSlice);
+    return narrativeCopy;
+  });
+  return result;
+}

package/src/id/generators.ts

@@ -0,0 +1,5 @@
+import { generateId } from '@auto-engineer/id';
+
+export function generateAutoId(): string {
+  return generateId({ prefix: 'AUTO-' });
+}