@auto-engineer/narrative 0.11.11

package/src/transformers/narrative-to-model/guards.ts

@@ -0,0 +1,24 @@
+export function hasDestination(d: unknown): d is { destination: unknown } {
+  return typeof d === 'object' && d !== null && 'destination' in d;
+}
+
+export function hasOrigin(d: unknown): d is { origin: unknown } {
+  return typeof d === 'object' && d !== null && 'origin' in d;
+}
+
+export function hasWithState(d: unknown): d is { _withState: unknown } {
+  return typeof d === 'object' && d !== null && '_withState' in d;
+}
+
+export function isValidIntegration(
+  integration: unknown,
+): integration is { type: 'integration'; systems: string[]; message?: unknown } {
+  return (
+    typeof integration === 'object' &&
+    integration !== null &&
+    'type' in integration &&
+    integration.type === 'integration' &&
+    'systems' in integration &&
+    Array.isArray(integration.systems)
+  );
+}

package/src/transformers/narrative-to-model/index.ts

@@ -0,0 +1,274 @@
+import { Narrative, Message, Model, Slice } from '../../index';
+import { globalIntegrationRegistry } from '../../integration-registry';
+import { integrationExportRegistry } from '../../integration-export-registry';
+import { TypeInfo } from '../../loader/ts-utils';
+import { resolveInferredType } from './type-inference';
+import { inlineAllMessageFieldTypes } from './inlining';
+import { ExampleShapeHints, applyExampleShapeHints } from './example-shapes';
+import { extractMessagesFromIntegrations, processDataItemIntegrations } from './integrations';
+import { matchesNarrativePattern } from './strings';
+import { assembleSpecs } from './assemble';
+import { processGiven, processWhen, processThen } from './spec-processors';
+
+type TypeResolver = (
+  t: string,
+  expected?: 'command' | 'event' | 'state',
+  exampleData?: unknown,
+) => { resolvedName: string; typeInfo: TypeInfo | undefined };
+
+function buildUnionTypes(typesByFile?: Map<string, Map<string, TypeInfo>>): Map<string, TypeInfo> | undefined {
+  if (!typesByFile) return undefined;
+  const u = new Map<string, TypeInfo>();
+  for (const [, m] of typesByFile) for (const [k, v] of m) u.set(k, v);
+  return u.size ? u : undefined;
+}
+
+function getTypesForNarrative(
+  flow: Narrative,
+  typesByFile?: Map<string, Map<string, TypeInfo>>,
+): Map<string, TypeInfo> | undefined {
+  if (!typesByFile) return undefined;
+
+  const sf = (flow as Record<string, unknown>).sourceFile as string | undefined;
+  if (typeof sf === 'string') {
+    const exact = typesByFile.get(sf) || typesByFile.get(sf.replace(/\\/g, '/'));
+    if (exact && exact.size > 0) return exact;
+  }
+
+  for (const [filePath, fileTypes] of typesByFile) {
+    const fileName = filePath.toLowerCase();
+    if (matchesNarrativePattern(fileName, flow.name)) {
+      return fileTypes;
+    }
+  }
+  return undefined;
+}
+
+function tryResolveFromNarrativeTypes(
+  t: string,
+  narrativeSpecificTypes: Map<string, TypeInfo>,
+  expected?: 'command' | 'event' | 'state',
+  exampleData?: unknown,
+): { resolvedName: string; typeInfo: TypeInfo | undefined } {
+  if (t !== 'InferredType') {
+    const typeInfo = narrativeSpecificTypes.get(t);
+    if (typeInfo) {
+      return { resolvedName: t, typeInfo };
+    }
+    const inferredName = resolveInferredType(t, narrativeSpecificTypes, expected, exampleData);
+    return { resolvedName: inferredName, typeInfo: narrativeSpecificTypes.get(inferredName) };
+  }
+
+  const inferredName = resolveInferredType(t, narrativeSpecificTypes, expected, exampleData);
+  return { resolvedName: inferredName, typeInfo: narrativeSpecificTypes.get(inferredName) };
+}
+
+function tryFallbackToUnionTypes(
+  t: string,
+  resolvedName: string,
+  typeInfo: TypeInfo | undefined,
+  unionTypes: Map<string, TypeInfo>,
+  expected?: 'command' | 'event' | 'state',
+  exampleData?: unknown,
+): { resolvedName: string; typeInfo: TypeInfo | undefined } {
+  if (resolvedName !== 'InferredType' && typeInfo) {
+    return { resolvedName, typeInfo };
+  }
+
+  const fallbackName = resolveInferredType(t, unionTypes, expected, exampleData);
+  const fallbackTypeInfo = unionTypes.get(fallbackName);
+
+  if (fallbackName !== 'InferredType' && fallbackTypeInfo) {
+    return { resolvedName: fallbackName, typeInfo: fallbackTypeInfo };
+  }
+
+  return { resolvedName, typeInfo };
+}
+
+function tryResolveFromUnionTypes(
+  t: string,
+  unionTypes: Map<string, TypeInfo>,
+  expected?: 'command' | 'event' | 'state',
+  exampleData?: unknown,
+): { resolvedName: string; typeInfo: TypeInfo | undefined } {
+  if (t !== 'InferredType') {
+    const typeInfo = unionTypes.get(t);
+    if (typeInfo) {
+      return { resolvedName: t, typeInfo };
+    }
+    const inferredName = resolveInferredType(t, unionTypes, expected, exampleData);
+    return { resolvedName: inferredName, typeInfo: unionTypes.get(inferredName) };
+  }
+
+  const inferredName = resolveInferredType(t, unionTypes, expected, exampleData);
+  return { resolvedName: inferredName, typeInfo: unionTypes.get(inferredName) };
+}
+
+function createTypeResolver(
+  narrativeSpecificTypes: Map<string, TypeInfo> | undefined,
+  unionTypes: Map<string, TypeInfo> | undefined,
+) {
+  return (
+    t: string,
+    expected?: 'command' | 'event' | 'state',
+    exampleData?: unknown,
+  ): { resolvedName: string; typeInfo: TypeInfo | undefined } => {
+    if (narrativeSpecificTypes) {
+      const result = tryResolveFromNarrativeTypes(t, narrativeSpecificTypes, expected, exampleData);
+      if (unionTypes) {
+        return tryFallbackToUnionTypes(t, result.resolvedName, result.typeInfo, unionTypes, expected, exampleData);
+      }
+      return result;
+    }
+
+    if (unionTypes) {
+      return tryResolveFromUnionTypes(t, unionTypes, expected, exampleData);
+    }
+
+    return { resolvedName: t, typeInfo: undefined };
+  };
+}
+
+function getSliceSpecs(slice: Slice) {
+  if ('server' in slice && slice.server?.specs !== undefined) {
+    return slice.server.specs;
+  } else if ('client' in slice && slice.client?.specs !== undefined) {
+    return slice.client.specs;
+  }
+  return undefined;
+}
+
+function processSliceSpecs(
+  slice: Slice,
+  resolveTypeAndInfo: TypeResolver,
+  messages: Map<string, Message>,
+  exampleShapeHints: ExampleShapeHints,
+): void {
+  const spec = getSliceSpecs(slice);
+
+  if (spec !== undefined && Array.isArray(spec.rules)) {
+    spec.rules.forEach((rule: unknown) => {
+      // Only process rule objects (server specs), not string rules (client specs)
+      if (
+        typeof rule === 'object' &&
+        rule !== null &&
+        'examples' in rule &&
+        Array.isArray((rule as { examples: unknown[] }).examples)
+      ) {
+        const ruleObj = rule as { examples: { given?: unknown; when?: unknown; then?: unknown }[] };
+        ruleObj.examples.forEach((example) => {
+          if (example.given !== undefined && example.given !== null) {
+            const givenArray = Array.isArray(example.given) ? example.given : [example.given];
+            processGiven(givenArray, resolveTypeAndInfo, messages, exampleShapeHints);
+          }
+          if (example.when !== undefined && example.when !== null) {
+            const whenArray = Array.isArray(example.when) ? example.when : [example.when];
+            processWhen(whenArray, slice, resolveTypeAndInfo, messages, exampleShapeHints);
+          }
+          if (example.then !== undefined && example.then !== null) {
+            const thenArray = Array.isArray(example.then) ? example.then : [example.then];
+            processThen(thenArray, resolveTypeAndInfo, messages, exampleShapeHints);
+          }
+        });
+      }
+    });
+  }
+}
+
+function processSliceIntegrations(
+  slice: Slice,
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+  messages: Map<string, Message>,
+): void {
+  // Integrations: from data & via
+  if ('server' in slice && slice.server != null && 'data' in slice.server && slice.server.data !== undefined) {
+    slice.server.data.forEach((d: unknown) => {
+      processDataItemIntegrations(d, integrations, messages);
+    });
+  }
+  if ('via' in slice && slice.via) {
+    slice.via.forEach((integrationName: string) => {
+      if (!integrations.has(integrationName)) {
+        const sourcePath = integrationExportRegistry.getSourcePath(integrationName);
+        if (sourcePath !== null && sourcePath !== undefined && sourcePath !== '') {
+          integrations.set(integrationName, {
+            name: integrationName,
+            description: `${integrationName} integration`,
+            source: sourcePath,
+          });
+        }
+      }
+    });
+  }
+}
+
+function processNarrative(
+  flow: Narrative,
+  getNarrativeSpecificTypes: (narrative: Narrative) => Map<string, TypeInfo> | undefined,
+  unionTypes: Map<string, TypeInfo> | undefined,
+  messages: Map<string, Message>,
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+  exampleShapeHints: ExampleShapeHints,
+): void {
+  const narrativeSpecificTypes = getNarrativeSpecificTypes(flow);
+  const resolveTypeAndInfo = createTypeResolver(narrativeSpecificTypes, unionTypes);
+
+  flow.slices.forEach((slice: Narrative['slices'][number]) => {
+    processSliceSpecs(slice, resolveTypeAndInfo, messages, exampleShapeHints);
+    processSliceIntegrations(slice, integrations, messages);
+  });
+}
+
+export const narrativesToModel = (narratives: Narrative[], typesByFile?: Map<string, Map<string, TypeInfo>>): Model => {
+  const messages = new Map<string, Message>();
+  const integrations = new Map<
+    string,
+    {
+      name: string;
+      description?: string;
+      source: string;
+    }
+  >();
+  const exampleShapeHints: ExampleShapeHints = new Map();
+
+  // Pull messages defined by registered integrations first
+  const registeredIntegrations = globalIntegrationRegistry.getAll();
+  const integrationMessages = extractMessagesFromIntegrations(registeredIntegrations);
+  for (const msg of integrationMessages) {
+    if (!messages.has(msg.name)) messages.set(msg.name, msg);
+  }
+
+  // Build a union of all discovered types (global fallback across files)
+  const unionTypes = buildUnionTypes(typesByFile);
+
+  // pick the best map for a given flow
+  const getNarrativeSpecificTypes = (narrative: Narrative): Map<string, TypeInfo> | undefined => {
+    return getTypesForNarrative(narrative, typesByFile);
+  };
+
+  narratives.forEach((narrative) =>
+    processNarrative(narrative, getNarrativeSpecificTypes, unionTypes, messages, integrations, exampleShapeHints),
+  );
+  // Ensure all registered integrations are listed
+  for (const integration of registeredIntegrations) {
+    const exportName = integrationExportRegistry.getExportNameForIntegration(integration);
+    if (!integrations.has(exportName)) {
+      const sourcePath = integrationExportRegistry.getSourcePath(exportName);
+      if (sourcePath !== null && sourcePath !== undefined && sourcePath !== '') {
+        integrations.set(exportName, {
+          name: exportName,
+          description: `${exportName} integration`,
+          source: sourcePath,
+        });
+      }
+    }
+  }
+  // Apply example-driven structural shapes (e.g., Array<Product> -> Array<{ ... }>)
+  applyExampleShapeHints(messages, exampleShapeHints);
+  // Then inline resolvable identifiers via TypeInfo (if available)
+  if (unionTypes) {
+    inlineAllMessageFieldTypes(messages, unionTypes);
+  }
+
+  return assembleSpecs(narratives, Array.from(messages.values()), Array.from(integrations.values()));
+};

package/src/transformers/narrative-to-model/inlining.ts

@@ -0,0 +1,106 @@
+import { TypeInfo } from '../../loader/ts-utils';
+import { Message } from '../../index';
+
+function inlineIdentifier(name: string, lookup: Map<string, TypeInfo>, seen = new Set<string>()): string | undefined {
+  // avoid infinite recursion on cyclical type graphs
+  if (seen.has(name)) return undefined;
+  seen.add(name);
+
+  const ti = lookup.get(name);
+  if (!ti) return undefined;
+
+  // If the referenced type has object-like fields, inline them as { a: T; b?: U }
+  if (Array.isArray(ti.dataFields) && ti.dataFields.length > 0) {
+    const parts = ti.dataFields.map((f) => {
+      const nested = inlineTypeString(f.type, lookup, seen);
+      const typeStr = nested ?? f.type;
+      return `${f.name}${f.required ? '' : '?'}: ${typeStr}`;
+    });
+    return `{ ${parts.join('; ')} }`;
+  }
+
+  // If we don't have field info, we can't inline safely
+  return undefined;
+}
+
+function handleArrayType(type: string, lookup: Map<string, TypeInfo>, seen: Set<string>): string | undefined {
+  const arr = type.match(/^Array<\s*([^>]+)\s*>$/);
+  if (arr) {
+    const inner = arr[1].trim();
+    const inlinedInner = inlineTypeString(inner, lookup, seen) ?? inlineIdentifier(inner, lookup, seen);
+    if (inlinedInner !== undefined && inlinedInner !== null) return `Array<${inlinedInner}>`;
+  }
+  return undefined;
+}
+
+function isPrimitiveType(type: string): boolean {
+  return (
+    type === 'string' ||
+    type === 'number' ||
+    type === 'boolean' ||
+    type === 'Date' ||
+    type === 'unknown' ||
+    type === 'any'
+  );
+}
+
+function isStructuralType(type: string): boolean {
+  const trimmed = type.trim();
+  return (trimmed.startsWith('{') && trimmed.endsWith('}')) || trimmed.includes(' | null') || trimmed.includes('|null');
+}
+
+function inlineTypeString(type: string, lookup: Map<string, TypeInfo>, seen = new Set<string>()): string | undefined {
+  // Array<T>
+  const arrayResult = handleArrayType(type, lookup, seen);
+  if (arrayResult !== undefined) return arrayResult;
+
+  // Basic primitives we don't touch
+  if (isPrimitiveType(type)) {
+    return undefined;
+  }
+
+  // Already structural? (starts with '{' and ends with '}' or looks like a union with null)
+  if (isStructuralType(type)) {
+    return undefined; // keep as is
+  }
+
+  // Handle union types like "string | null"
+  if (type.includes(' | ')) {
+    const parts = type.split(' | ').map((p) => p.trim());
+    const inlinedParts = parts.map((part) => {
+      if (isPrimitiveType(part) || part === 'null') {
+        return part;
+      }
+      const inlined = inlineTypeString(part, lookup, seen);
+      return inlined ?? part;
+    });
+    // Fix: Convert "unknown" back to "null" in union types (TypeScript parsing issue)
+    // Only replace unknown with null when it's part of a union with other types
+    const result = inlinedParts.join(' | ');
+    if (inlinedParts.length > 1 && inlinedParts.includes('unknown')) {
+      return result.replace(/\bunknown\b/g, 'null');
+    }
+    return result;
+  }
+
+  // Single identifier? Try to inline it
+  const ident = type.match(/^[A-Za-z_]\w*$/);
+  if (ident) {
+    const asObj = inlineIdentifier(type, lookup, seen);
+    if (asObj !== undefined) return asObj;
+    return undefined;
+  }
+
+  // Generic we don't specially handle (Record<...>, etc.) -> leave as-is
+  return undefined;
+}
+
+export function inlineAllMessageFieldTypes(msgs: Map<string, Message>, lookup: Map<string, TypeInfo>): void {
+  for (const [, msg] of msgs) {
+    if (!Array.isArray(msg.fields)) continue;
+    msg.fields = msg.fields.map((f) => {
+      const inlined = inlineTypeString(f.type, lookup);
+      return inlined !== undefined ? { ...f, type: inlined } : f;
+    });
+  }
+}

package/src/transformers/narrative-to-model/integrations.ts

@@ -0,0 +1,201 @@
+import { Integration } from '../../types';
+import { Message } from '../../index';
+import { zodSchemaToFields } from './zod-adapter';
+import { debugIntegrations } from './debug';
+import { hasDestination, hasOrigin, hasWithState, isValidIntegration } from './guards';
+import { processDestinationMessage } from './messages';
+import { integrationExportRegistry } from '../../integration-export-registry';
+
+const integrationImportPaths = new Map<string, string>();
+
+export function addIntegrationImportPath(integrationName: string, importPath: string): void {
+  integrationImportPaths.set(integrationName, importPath);
+}
+
+export function addIntegrationToMap(
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+  system: string,
+): void {
+  if (!integrations.has(system)) {
+    let sourcePath = integrationImportPaths.get(system);
+    if (sourcePath === null || sourcePath === undefined || sourcePath === '') {
+      sourcePath = integrationExportRegistry.getSourcePath(system);
+    }
+
+    if (sourcePath !== null && sourcePath !== undefined && sourcePath !== '') {
+      integrations.set(system, {
+        name: system,
+        description: `${system} integration`,
+        source: sourcePath,
+      });
+    }
+  }
+}
+
+export function processDestination(
+  d: { destination: unknown },
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+  messages: Map<string, Message>,
+): void {
+  if (isValidIntegration(d.destination)) {
+    d.destination.systems.forEach((system: string) => {
+      addIntegrationToMap(integrations, system);
+      if (
+        typeof d.destination === 'object' &&
+        d.destination !== null &&
+        'message' in d.destination &&
+        d.destination.message !== null
+      ) {
+        processDestinationMessage(d.destination.message, messages);
+      }
+    });
+  }
+}
+
+export function processOrigin(
+  d: { origin: unknown },
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+): void {
+  if (isValidIntegration(d.origin)) {
+    d.origin.systems.forEach((system: string) => {
+      addIntegrationToMap(integrations, system);
+    });
+  }
+}
+
+export function processWithStateOrigin(
+  d: { _withState: unknown },
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+): void {
+  if (typeof d._withState === 'object' && d._withState !== null && 'origin' in d._withState) {
+    const withState = d._withState as { origin: unknown };
+    if (isValidIntegration(withState.origin)) {
+      withState.origin.systems.forEach((system: string) => {
+        addIntegrationToMap(integrations, system);
+      });
+    }
+  }
+}
+
+export function processDataItemIntegrations(
+  d: unknown,
+  integrations: Map<string, { name: string; description?: string; source: string }>,
+  messages: Map<string, Message>,
+): void {
+  if (hasDestination(d)) {
+    processDestination(d, integrations, messages);
+  }
+
+  if (hasOrigin(d)) {
+    processOrigin(d, integrations);
+  }
+
+  if (hasWithState(d)) {
+    processWithStateOrigin(d, integrations);
+  }
+}
+
+function flattenStateEnvelope(fields: Message['fields']): Message['fields'] {
+  const dataIdx = fields.findIndex(
+    (f) => f.name === 'data' && typeof f.type === 'string' && f.type.trim().startsWith('{'),
+  );
+
+  if (dataIdx !== -1) {
+    const src = fields[dataIdx].type; // "{ a: T, b?: U }" or "{ a: T; b?: U }"
+    const bodyMatch = src.match(/^\{\s*([^}]*)\s*\}$/);
+    if (bodyMatch && bodyMatch[1]) {
+      const body = bodyMatch[1];
+      const inner: { name: string; type: string; required: boolean }[] = [];
+      // support comma OR semicolon separators
+      const re = /(\w+)(\?)?\s*:\s*([^;,]+)(?=[;,]|$)/g;
+      let m: RegExpExecArray | null;
+      while ((m = re.exec(body)) !== null) {
+        inner.push({ name: m[1], required: !m[2], type: m[3].trim() });
+      }
+      return fields
+        .filter((f) => f.name !== 'type' && f.name !== 'data')
+        .concat(inner.map((f) => ({ ...f, required: f.required })));
+    }
+  }
+
+  // No envelope? still drop the discriminator 'type'
+  return fields.filter((f) => f.name !== 'type');
+}
+
+function createMessageFromFields(
+  name: string,
+  fields: Message['fields'],
+  messageType: 'command' | 'state' | 'event',
+): Message {
+  if (messageType === 'event') {
+    return {
+      type: 'event',
+      name,
+      fields,
+      source: 'external',
+      metadata: { version: 1 },
+    };
+  } else if (messageType === 'command') {
+    return {
+      type: 'command',
+      name,
+      fields,
+      metadata: { version: 1 },
+    };
+  } else {
+    return {
+      type: 'state',
+      name,
+      fields,
+      metadata: { version: 1 },
+    };
+  }
+}
+
+const extractSchemaType = (
+  integration: Integration,
+  schemaType: 'Commands' | 'Queries' | 'Reactions',
+  messageType: 'command' | 'state' | 'event',
+): Message[] => {
+  const messages: Message[] = [];
+  const schema = integration[schemaType]?.schema;
+
+  if (schema) {
+    debugIntegrations(`[extractMessagesFromIntegrations] Found ${schemaType}.schema:`, Object.keys(schema));
+    for (const [name, schemaItem] of Object.entries(schema)) {
+      if (schemaItem) {
+        let fields = zodSchemaToFields(schemaItem);
+
+        // Flatten state envelope { type, data: { ... } } for integration-derived states
+        if (messageType === 'state') {
+          fields = flattenStateEnvelope(fields);
+        }
+
+        debugIntegrations(
+          `[extractMessagesFromIntegrations] Creating ${messageType} message '${name}' with fields:`,
+          fields,
+        );
+
+        messages.push(createMessageFromFields(name, fields, messageType));
+      }
+    }
+  } else {
+    debugIntegrations(`[extractMessagesFromIntegrations] No ${schemaType}.schema found for ${integration.name}`);
+  }
+
+  return messages;
+};
+
+export const extractMessagesFromIntegrations = (integrations: Integration[]): Message[] => {
+  const messages: Message[] = [];
+
+  for (const integration of integrations) {
+    debugIntegrations('Processing integration for message extraction: %s', integration.name);
+
+    messages.push(...extractSchemaType(integration, 'Commands', 'command'));
+    messages.push(...extractSchemaType(integration, 'Queries', 'state'));
+    messages.push(...extractSchemaType(integration, 'Reactions', 'event'));
+  }
+
+  return messages;
+};