@steno-ai/engine 0.1.16 → 0.1.18

package/src/extraction/structured-extractor.ts

@@ -0,0 +1,471 @@
+/**
+ * Structured data extractor — bypasses LLM entirely.
+ *
+ * Handles structured_event, structured_task, structured_email, structured_vault
+ * input types by directly creating entities, edges, and facts from known fields.
+ * Zero LLM cost, deterministic, high confidence.
+ */
+
+import type { ExtractionResult, ExtractedFact, ExtractedEntity, ExtractedEdge } from './types.js';
+import type { SourceType, EdgeType } from '../config.js';
+
+// ---------------------------------------------------------------------------
+// Structured input schemas
+// ---------------------------------------------------------------------------
+
+export interface StructuredEvent {
+  title: string;
+  startTime: string;          // ISO 8601
+  endTime?: string;
+  location?: string;
+  description?: string;
+  organizers?: string[];       // org/person names
+  attendees?: string[];
+  url?: string;
+  provider?: string;           // 'google' | 'microsoft' | 'partiful' etc.
+  externalId?: string;         // calendar event ID, vault item ID, etc.
+  sourceType?: 'calendar' | 'vault';
+}
+
+export interface StructuredTask {
+  title: string;
+  description?: string;
+  status?: string;
+  priority?: string;
+  category?: string;
+  dueDate?: string;            // ISO 8601
+  tags?: string[];
+  externalId?: string;
+}
+
+export interface StructuredEmail {
+  subject: string;
+  from: string;
+  to?: string[];
+  body?: string;               // truncated
+  date: string;                // ISO 8601
+  isUnread?: boolean;
+  threadId?: string;
+  provider?: string;           // 'gmail' | 'outlook'
+  externalId?: string;
+}
+
+export interface StructuredVault {
+  title: string;
+  contentType: string;         // 'event', 'article', 'job', 'recipe', etc.
+  url?: string;
+  source?: string;             // domain
+  savedAt: string;             // ISO 8601
+  content?: string;            // truncated page content
+  metadata?: Record<string, unknown>;
+  externalId?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function canonicalize(name: string): string {
+  return name.toLowerCase().replace(/[^a-z0-9\s.-]/g, '').replace(/\s+/g, ' ').trim();
+}
+
+function formatDate(iso: string): string {
+  try {
+    return new Date(iso).toLocaleDateString('en-US', {
+      weekday: 'long', year: 'numeric', month: 'long', day: 'numeric',
+    });
+  } catch {
+    return iso;
+  }
+}
+
+function formatTime(iso: string): string {
+  try {
+    return new Date(iso).toLocaleTimeString('en-US', {
+      hour: 'numeric', minute: '2-digit', hour12: true,
+    });
+  } catch {
+    return '';
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Extractors
+// ---------------------------------------------------------------------------
+
+export function extractStructuredEvent(data: StructuredEvent): ExtractionResult {
+  if (!data?.title) return { facts: [], entities: [], edges: [], tier: 'heuristic', confidence: 1.0, tokensInput: 0, tokensOutput: 0, model: null };
+
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  // Main event entity
+  const eventCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: 'event',
+    canonicalName: eventCanonical,
+    properties: {
+      startTime: data.startTime,
+      endTime: data.endTime,
+      location: data.location,
+      url: data.url,
+      provider: data.provider,
+      externalId: data.externalId,
+      sourceType: data.sourceType,
+    },
+  });
+
+  // Location entity
+  if (data.location) {
+    const locCanonical = canonicalize(data.location);
+    entities.push({
+      name: data.location,
+      entityType: 'location',
+      canonicalName: locCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: eventCanonical,
+      targetName: locCanonical,
+      relation: 'located_at',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Organizer entities
+  for (const org of data.organizers ?? []) {
+    const orgCanonical = canonicalize(org);
+    entities.push({
+      name: org,
+      entityType: 'organization',
+      canonicalName: orgCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: eventCanonical,
+      targetName: orgCanonical,
+      relation: 'hosted_by',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Attendee entities
+  for (const attendee of data.attendees ?? []) {
+    const attCanonical = canonicalize(attendee);
+    entities.push({
+      name: attendee,
+      entityType: 'person',
+      canonicalName: attCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: attCanonical,
+      targetName: eventCanonical,
+      relation: 'attends',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Build fact content
+  let factContent = `Event: "${data.title}" on ${formatDate(data.startTime)}`;
+  if (data.startTime) factContent += ` at ${formatTime(data.startTime)}`;
+  if (data.endTime) factContent += ` - ${formatTime(data.endTime)}`;
+  if (data.location) factContent += ` at ${data.location}`;
+  if (data.organizers?.length) factContent += `. Hosted by ${data.organizers.join(', ')}`;
+  if (data.description) factContent += `. ${data.description.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: 0.8,
+    confidence: 1.0,
+    sourceType: (data.sourceType === 'vault' ? 'structured_vault' : 'structured_event') as SourceType,
+    modality: 'text',
+    tags: ['structured', 'event', ...(data.provider ? [data.provider] : [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [eventCanonical, ...entities.filter(e => e.canonicalName !== eventCanonical).map(e => e.canonicalName)],
+    eventDate: new Date(data.startTime),
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredTask(data: StructuredTask): ExtractionResult {
+  if (!data?.title) return { facts: [], entities: [], edges: [], tier: 'heuristic', confidence: 1.0, tokensInput: 0, tokensOutput: 0, model: null };
+
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  const taskCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: 'task',
+    canonicalName: taskCanonical,
+    properties: {
+      status: data.status,
+      priority: data.priority,
+      category: data.category,
+      dueDate: data.dueDate,
+      externalId: data.externalId,
+    },
+  });
+
+  // Category entity
+  if (data.category) {
+    const catCanonical = canonicalize(data.category);
+    entities.push({
+      name: data.category,
+      entityType: 'topic',
+      canonicalName: catCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: taskCanonical,
+      targetName: catCanonical,
+      relation: 'categorized_as',
+      edgeType: 'hierarchical',
+      confidence: 1.0,
+    });
+  }
+
+  let factContent = `Task: "${data.title}"`;
+  if (data.status) factContent += ` (${data.status})`;
+  if (data.priority) factContent += `, priority: ${data.priority}`;
+  if (data.dueDate) factContent += `, due ${formatDate(data.dueDate)}`;
+  if (data.description) factContent += `. ${data.description.slice(0, 200)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: data.priority === 'high' || data.priority === 'urgent' ? 0.9 : 0.7,
+    confidence: 1.0,
+    sourceType: 'structured_task' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'task', ...(data.tags ?? [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [taskCanonical],
+    eventDate: data.dueDate ? new Date(data.dueDate) : undefined,
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredEmail(data: StructuredEmail): ExtractionResult {
+  if (!data?.subject && !data?.from) return { facts: [], entities: [], edges: [], tier: 'heuristic', confidence: 1.0, tokensInput: 0, tokensOutput: 0, model: null };
+
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  // Sender entity
+  const senderCanonical = canonicalize(data.from);
+  entities.push({
+    name: data.from,
+    entityType: 'person',
+    canonicalName: senderCanonical,
+    properties: { email: data.from },
+  });
+
+  // Subject as topic entity if substantial
+  if (data.subject && data.subject.length > 5) {
+    const subjectCanonical = canonicalize(data.subject);
+    entities.push({
+      name: data.subject,
+      entityType: 'topic',
+      canonicalName: subjectCanonical,
+      properties: { threadId: data.threadId, provider: data.provider },
+    });
+    edges.push({
+      sourceName: senderCanonical,
+      targetName: subjectCanonical,
+      relation: 'authored',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Recipients
+  for (const to of data.to ?? []) {
+    const toCanonical = canonicalize(to);
+    entities.push({
+      name: to,
+      entityType: 'person',
+      canonicalName: toCanonical,
+      properties: { email: to },
+    });
+  }
+
+  let factContent = `Email from ${data.from}: "${data.subject}"`;
+  if (data.date) factContent += ` on ${formatDate(data.date)}`;
+  if (data.body) factContent += `. ${data.body.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: data.isUnread ? 0.8 : 0.5,
+    confidence: 1.0,
+    sourceType: 'structured_email' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'email', ...(data.provider ? [data.provider] : []), ...(data.isUnread ? ['unread'] : [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [senderCanonical],
+    eventDate: new Date(data.date),
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredVault(data: StructuredVault): ExtractionResult {
+  if (!data?.title) return { facts: [], entities: [], edges: [], tier: 'heuristic', confidence: 1.0, tokensInput: 0, tokensOutput: 0, model: null };
+
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  const vaultCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: data.contentType === 'event' ? 'event' : 'topic',
+    canonicalName: vaultCanonical,
+    properties: {
+      contentType: data.contentType,
+      url: data.url,
+      source: data.source,
+      savedAt: data.savedAt,
+      externalId: data.externalId,
+      ...(data.metadata ?? {}),
+    },
+  });
+
+  // Source domain entity
+  if (data.source) {
+    const sourceCanonical = canonicalize(data.source);
+    entities.push({
+      name: data.source,
+      entityType: 'source',
+      canonicalName: sourceCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: vaultCanonical,
+      targetName: sourceCanonical,
+      relation: 'saved_from',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // If event type, extract organizers from metadata
+  const organizers = data.metadata?.organizer || data.metadata?.organizers;
+  if (organizers) {
+    const orgList = typeof organizers === 'string'
+      ? organizers.split(/,\s*|(?:\s+and\s+)/)
+      : Array.isArray(organizers) ? organizers : [];
+    for (const org of orgList) {
+      const trimmed = (org as string).trim();
+      if (!trimmed) continue;
+      const orgCanonical = canonicalize(trimmed);
+      entities.push({
+        name: trimmed,
+        entityType: 'organization',
+        canonicalName: orgCanonical,
+        properties: {},
+      });
+      edges.push({
+        sourceName: vaultCanonical,
+        targetName: orgCanonical,
+        relation: 'hosted_by',
+        edgeType: 'associative',
+        confidence: 1.0,
+      });
+    }
+  }
+
+  let factContent = `Saved to vault: "${data.title}" (${data.contentType})`;
+  if (data.source) factContent += ` from ${data.source}`;
+  if (data.savedAt) factContent += ` on ${formatDate(data.savedAt)}`;
+  if (data.content) factContent += `. ${data.content.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: 0.7,
+    confidence: 1.0,
+    sourceType: 'structured_vault' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'vault', data.contentType],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [vaultCanonical, ...entities.filter(e => e.canonicalName !== vaultCanonical).map(e => e.canonicalName)],
+    eventDate: data.metadata?.date ? new Date(data.metadata.date as string) : undefined,
+    documentDate: new Date(data.savedAt),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Router — picks the right extractor based on inputType
+// ---------------------------------------------------------------------------
+
+const STRUCTURED_INPUT_TYPES = new Set([
+  'structured_event',
+  'structured_task',
+  'structured_email',
+  'structured_vault',
+]);
+
+export function isStructuredInput(inputType: string): boolean {
+  return STRUCTURED_INPUT_TYPES.has(inputType);
+}
+
+export function extractStructured(inputType: string, data: unknown): ExtractionResult {
+  switch (inputType) {
+    case 'structured_event':
+      return extractStructuredEvent(data as StructuredEvent);
+    case 'structured_task':
+      return extractStructuredTask(data as StructuredTask);
+    case 'structured_email':
+      return extractStructuredEmail(data as StructuredEmail);
+    case 'structured_vault':
+      return extractStructuredVault(data as StructuredVault);
+    default:
+      throw new Error(`Unknown structured input type: ${inputType}`);
+  }
+}

package/src/extraction/types.ts

@@ -60,7 +60,7 @@ export interface ExtractionInput {
   scope: Scope;
   scopeId: string;
   sessionId?: string;
-  inputType: 'conversation' | 'document' | 'url' | 'raw_text' | 'image' | 'audio' | 'code' | 'codebase_scan' | 'file_change' | 'architecture_doc';
+  inputType: 'conversation' | 'document' | 'url' | 'raw_text' | 'image' | 'audio' | 'code' | 'codebase_scan' | 'file_change' | 'architecture_doc' | 'structured_event' | 'structured_task' | 'structured_email' | 'structured_vault';
   data: unknown;
   existingFacts?: Array<{ id: string; lineageId: string; content: string; embedding?: number[] }>;
   /** Source provider for provenance tracking — where did this data come from? */