@steno-ai/engine 0.1.15 → 0.1.17

package/src/extraction/structured-extractor.ts

@@ -0,0 +1,463 @@
+/**
+ * Structured data extractor — bypasses LLM entirely.
+ *
+ * Handles structured_event, structured_task, structured_email, structured_vault
+ * input types by directly creating entities, edges, and facts from known fields.
+ * Zero LLM cost, deterministic, high confidence.
+ */
+
+import type { ExtractionResult, ExtractedFact, ExtractedEntity, ExtractedEdge } from './types.js';
+import type { SourceType, EdgeType } from '../config.js';
+
+// ---------------------------------------------------------------------------
+// Structured input schemas
+// ---------------------------------------------------------------------------
+
+export interface StructuredEvent {
+  title: string;
+  startTime: string;          // ISO 8601
+  endTime?: string;
+  location?: string;
+  description?: string;
+  organizers?: string[];       // org/person names
+  attendees?: string[];
+  url?: string;
+  provider?: string;           // 'google' | 'microsoft' | 'partiful' etc.
+  externalId?: string;         // calendar event ID, vault item ID, etc.
+  sourceType?: 'calendar' | 'vault';
+}
+
+export interface StructuredTask {
+  title: string;
+  description?: string;
+  status?: string;
+  priority?: string;
+  category?: string;
+  dueDate?: string;            // ISO 8601
+  tags?: string[];
+  externalId?: string;
+}
+
+export interface StructuredEmail {
+  subject: string;
+  from: string;
+  to?: string[];
+  body?: string;               // truncated
+  date: string;                // ISO 8601
+  isUnread?: boolean;
+  threadId?: string;
+  provider?: string;           // 'gmail' | 'outlook'
+  externalId?: string;
+}
+
+export interface StructuredVault {
+  title: string;
+  contentType: string;         // 'event', 'article', 'job', 'recipe', etc.
+  url?: string;
+  source?: string;             // domain
+  savedAt: string;             // ISO 8601
+  content?: string;            // truncated page content
+  metadata?: Record<string, unknown>;
+  externalId?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function canonicalize(name: string): string {
+  return name.toLowerCase().replace(/[^a-z0-9\s.-]/g, '').replace(/\s+/g, ' ').trim();
+}
+
+function formatDate(iso: string): string {
+  try {
+    return new Date(iso).toLocaleDateString('en-US', {
+      weekday: 'long', year: 'numeric', month: 'long', day: 'numeric',
+    });
+  } catch {
+    return iso;
+  }
+}
+
+function formatTime(iso: string): string {
+  try {
+    return new Date(iso).toLocaleTimeString('en-US', {
+      hour: 'numeric', minute: '2-digit', hour12: true,
+    });
+  } catch {
+    return '';
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Extractors
+// ---------------------------------------------------------------------------
+
+export function extractStructuredEvent(data: StructuredEvent): ExtractionResult {
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  // Main event entity
+  const eventCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: 'event',
+    canonicalName: eventCanonical,
+    properties: {
+      startTime: data.startTime,
+      endTime: data.endTime,
+      location: data.location,
+      url: data.url,
+      provider: data.provider,
+      externalId: data.externalId,
+      sourceType: data.sourceType,
+    },
+  });
+
+  // Location entity
+  if (data.location) {
+    const locCanonical = canonicalize(data.location);
+    entities.push({
+      name: data.location,
+      entityType: 'location',
+      canonicalName: locCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: eventCanonical,
+      targetName: locCanonical,
+      relation: 'located_at',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Organizer entities
+  for (const org of data.organizers ?? []) {
+    const orgCanonical = canonicalize(org);
+    entities.push({
+      name: org,
+      entityType: 'organization',
+      canonicalName: orgCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: eventCanonical,
+      targetName: orgCanonical,
+      relation: 'hosted_by',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Attendee entities
+  for (const attendee of data.attendees ?? []) {
+    const attCanonical = canonicalize(attendee);
+    entities.push({
+      name: attendee,
+      entityType: 'person',
+      canonicalName: attCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: attCanonical,
+      targetName: eventCanonical,
+      relation: 'attends',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Build fact content
+  let factContent = `Event: "${data.title}" on ${formatDate(data.startTime)}`;
+  if (data.startTime) factContent += ` at ${formatTime(data.startTime)}`;
+  if (data.endTime) factContent += ` - ${formatTime(data.endTime)}`;
+  if (data.location) factContent += ` at ${data.location}`;
+  if (data.organizers?.length) factContent += `. Hosted by ${data.organizers.join(', ')}`;
+  if (data.description) factContent += `. ${data.description.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: 0.8,
+    confidence: 1.0,
+    sourceType: (data.sourceType === 'vault' ? 'structured_vault' : 'structured_event') as SourceType,
+    modality: 'text',
+    tags: ['structured', 'event', ...(data.provider ? [data.provider] : [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [eventCanonical, ...entities.filter(e => e.canonicalName !== eventCanonical).map(e => e.canonicalName)],
+    eventDate: new Date(data.startTime),
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredTask(data: StructuredTask): ExtractionResult {
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  const taskCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: 'task',
+    canonicalName: taskCanonical,
+    properties: {
+      status: data.status,
+      priority: data.priority,
+      category: data.category,
+      dueDate: data.dueDate,
+      externalId: data.externalId,
+    },
+  });
+
+  // Category entity
+  if (data.category) {
+    const catCanonical = canonicalize(data.category);
+    entities.push({
+      name: data.category,
+      entityType: 'topic',
+      canonicalName: catCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: taskCanonical,
+      targetName: catCanonical,
+      relation: 'categorized_as',
+      edgeType: 'hierarchical',
+      confidence: 1.0,
+    });
+  }
+
+  let factContent = `Task: "${data.title}"`;
+  if (data.status) factContent += ` (${data.status})`;
+  if (data.priority) factContent += `, priority: ${data.priority}`;
+  if (data.dueDate) factContent += `, due ${formatDate(data.dueDate)}`;
+  if (data.description) factContent += `. ${data.description.slice(0, 200)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: data.priority === 'high' || data.priority === 'urgent' ? 0.9 : 0.7,
+    confidence: 1.0,
+    sourceType: 'structured_task' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'task', ...(data.tags ?? [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [taskCanonical],
+    eventDate: data.dueDate ? new Date(data.dueDate) : undefined,
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredEmail(data: StructuredEmail): ExtractionResult {
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  // Sender entity
+  const senderCanonical = canonicalize(data.from);
+  entities.push({
+    name: data.from,
+    entityType: 'person',
+    canonicalName: senderCanonical,
+    properties: { email: data.from },
+  });
+
+  // Subject as topic entity if substantial
+  if (data.subject && data.subject.length > 5) {
+    const subjectCanonical = canonicalize(data.subject);
+    entities.push({
+      name: data.subject,
+      entityType: 'topic',
+      canonicalName: subjectCanonical,
+      properties: { threadId: data.threadId, provider: data.provider },
+    });
+    edges.push({
+      sourceName: senderCanonical,
+      targetName: subjectCanonical,
+      relation: 'authored',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // Recipients
+  for (const to of data.to ?? []) {
+    const toCanonical = canonicalize(to);
+    entities.push({
+      name: to,
+      entityType: 'person',
+      canonicalName: toCanonical,
+      properties: { email: to },
+    });
+  }
+
+  let factContent = `Email from ${data.from}: "${data.subject}"`;
+  if (data.date) factContent += ` on ${formatDate(data.date)}`;
+  if (data.body) factContent += `. ${data.body.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: data.isUnread ? 0.8 : 0.5,
+    confidence: 1.0,
+    sourceType: 'structured_email' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'email', ...(data.provider ? [data.provider] : []), ...(data.isUnread ? ['unread'] : [])],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [senderCanonical],
+    eventDate: new Date(data.date),
+    documentDate: new Date(),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+export function extractStructuredVault(data: StructuredVault): ExtractionResult {
+  const entities: ExtractedEntity[] = [];
+  const edges: ExtractedEdge[] = [];
+
+  const vaultCanonical = canonicalize(data.title);
+  entities.push({
+    name: data.title,
+    entityType: data.contentType === 'event' ? 'event' : 'topic',
+    canonicalName: vaultCanonical,
+    properties: {
+      contentType: data.contentType,
+      url: data.url,
+      source: data.source,
+      savedAt: data.savedAt,
+      externalId: data.externalId,
+      ...(data.metadata ?? {}),
+    },
+  });
+
+  // Source domain entity
+  if (data.source) {
+    const sourceCanonical = canonicalize(data.source);
+    entities.push({
+      name: data.source,
+      entityType: 'source',
+      canonicalName: sourceCanonical,
+      properties: {},
+    });
+    edges.push({
+      sourceName: vaultCanonical,
+      targetName: sourceCanonical,
+      relation: 'saved_from',
+      edgeType: 'associative',
+      confidence: 1.0,
+    });
+  }
+
+  // If event type, extract organizers from metadata
+  const organizers = data.metadata?.organizer || data.metadata?.organizers;
+  if (organizers) {
+    const orgList = typeof organizers === 'string'
+      ? organizers.split(/,\s*|(?:\s+and\s+)/)
+      : Array.isArray(organizers) ? organizers : [];
+    for (const org of orgList) {
+      const trimmed = (org as string).trim();
+      if (!trimmed) continue;
+      const orgCanonical = canonicalize(trimmed);
+      entities.push({
+        name: trimmed,
+        entityType: 'organization',
+        canonicalName: orgCanonical,
+        properties: {},
+      });
+      edges.push({
+        sourceName: vaultCanonical,
+        targetName: orgCanonical,
+        relation: 'hosted_by',
+        edgeType: 'associative',
+        confidence: 1.0,
+      });
+    }
+  }
+
+  let factContent = `Saved to vault: "${data.title}" (${data.contentType})`;
+  if (data.source) factContent += ` from ${data.source}`;
+  if (data.savedAt) factContent += ` on ${formatDate(data.savedAt)}`;
+  if (data.content) factContent += `. ${data.content.slice(0, 300)}`;
+
+  const fact: ExtractedFact = {
+    content: factContent,
+    importance: 0.7,
+    confidence: 1.0,
+    sourceType: 'structured_vault' as SourceType,
+    modality: 'text',
+    tags: ['structured', 'vault', data.contentType],
+    originalContent: JSON.stringify(data),
+    entityCanonicalNames: [vaultCanonical, ...entities.filter(e => e.canonicalName !== vaultCanonical).map(e => e.canonicalName)],
+    eventDate: data.metadata?.date ? new Date(data.metadata.date as string) : undefined,
+    documentDate: new Date(data.savedAt),
+  };
+
+  return {
+    facts: [fact],
+    entities,
+    edges,
+    tier: 'heuristic',
+    confidence: 1.0,
+    tokensInput: 0,
+    tokensOutput: 0,
+    model: null,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Router — picks the right extractor based on inputType
+// ---------------------------------------------------------------------------
+
+const STRUCTURED_INPUT_TYPES = new Set([
+  'structured_event',
+  'structured_task',
+  'structured_email',
+  'structured_vault',
+]);
+
+export function isStructuredInput(inputType: string): boolean {
+  return STRUCTURED_INPUT_TYPES.has(inputType);
+}
+
+export function extractStructured(inputType: string, data: unknown): ExtractionResult {
+  switch (inputType) {
+    case 'structured_event':
+      return extractStructuredEvent(data as StructuredEvent);
+    case 'structured_task':
+      return extractStructuredTask(data as StructuredTask);
+    case 'structured_email':
+      return extractStructuredEmail(data as StructuredEmail);
+    case 'structured_vault':
+      return extractStructuredVault(data as StructuredVault);
+    default:
+      throw new Error(`Unknown structured input type: ${inputType}`);
+  }
+}

package/src/extraction/types.ts

@@ -60,11 +60,13 @@ export interface ExtractionInput {
   scope: Scope;
   scopeId: string;
   sessionId?: string;
-  inputType: 'conversation' | 'document' | 'url' | 'raw_text' | 'image' | 'audio' | 'code' | 'codebase_scan' | 'file_change' | 'architecture_doc';
+  inputType: 'conversation' | 'document' | 'url' | 'raw_text' | 'image' | 'audio' | 'code' | 'codebase_scan' | 'file_change' | 'architecture_doc' | 'structured_event' | 'structured_task' | 'structured_email' | 'structured_vault';
   data: unknown;
   existingFacts?: Array<{ id: string; lineageId: string; content: string; embedding?: number[] }>;
   /** Source provider for provenance tracking — where did this data come from? */
   sourceProvider?: 'user_direct' | 'email' | 'calendar' | 'slack' | 'github' | 'linear' | 'jira' | 'code' | 'document';
+  /** Opt-in: run cross-scope identity resolution after entity creation */
+  resolveIdentities?: boolean;
 }
 
 /** Final output of the full pipeline */

package/src/identity/index.ts

@@ -0,0 +1 @@
+export { resolveIdentities, type IdentityCandidate, type AliasCandidate } from './resolver.js';

package/src/identity/resolver.ts

@@ -0,0 +1,149 @@
+export interface IdentityCandidate {
+  id: string;
+  canonicalName: string;
+  entityType: string;
+  properties: Record<string, unknown>;
+}
+
+export interface AliasCandidate {
+  primaryEntityId: string;
+  aliasEntityId: string;
+  confidence: number;
+  matchReason: 'email_match' | 'fuzzy_name' | 'temporal_cooccurrence' | 'manual';
+}
+
+/**
+ * Jaro-Winkler similarity algorithm.
+ * Returns a value between 0 (no similarity) and 1 (identical).
+ */
+export function jaroWinkler(s1: string, s2: string): number {
+  if (s1 === s2) return 1.0;
+  if (s1.length === 0 || s2.length === 0) return 0.0;
+
+  const matchDistance = Math.floor(Math.max(s1.length, s2.length) / 2) - 1;
+
+  const s1Matches = new Array(s1.length).fill(false);
+  const s2Matches = new Array(s2.length).fill(false);
+
+  let matches = 0;
+  let transpositions = 0;
+
+  // Find matching characters
+  for (let i = 0; i < s1.length; i++) {
+    const start = Math.max(0, i - matchDistance);
+    const end = Math.min(i + matchDistance + 1, s2.length);
+
+    for (let j = start; j < end; j++) {
+      if (s2Matches[j] || s1[i] !== s2[j]) continue;
+      s1Matches[i] = true;
+      s2Matches[j] = true;
+      matches++;
+      break;
+    }
+  }
+
+  if (matches === 0) return 0.0;
+
+  // Count transpositions
+  let k = 0;
+  for (let i = 0; i < s1.length; i++) {
+    if (!s1Matches[i]) continue;
+    while (!s2Matches[k]) k++;
+    if (s1[i] !== s2[k]) transpositions++;
+    k++;
+  }
+
+  const jaro =
+    (matches / s1.length + matches / s2.length + (matches - transpositions / 2) / matches) / 3;
+
+  // Winkler prefix bonus (up to 4 common prefix chars)
+  let prefix = 0;
+  for (let i = 0; i < Math.min(4, Math.min(s1.length, s2.length)); i++) {
+    if (s1[i] === s2[i]) prefix++;
+    else break;
+  }
+
+  return jaro + prefix * 0.1 * (1 - jaro);
+}
+
+/**
+ * Normalize a name for fuzzy comparison:
+ * lowercase and strip separators (. _ - @).
+ */
+export function normalizeName(name: string): string {
+  return name.toLowerCase().replace(/[._\-@]/g, '');
+}
+
+/**
+ * Resolve identities across a list of entity candidates using a three-tier strategy:
+ *  1. Exact email match (confidence 1.0)
+ *  2. Fuzzy canonical name match via Jaro-Winkler (confidence 0.6–0.8)
+ *
+ * Returns alias pairs without duplicates.
+ */
+export function resolveIdentities(entities: IdentityCandidate[]): AliasCandidate[] {
+  const aliases: AliasCandidate[] = [];
+  const seen = new Set<string>();
+
+  const addAlias = (
+    primaryEntityId: string,
+    aliasEntityId: string,
+    confidence: number,
+    matchReason: AliasCandidate['matchReason'],
+  ): void => {
+    const key = [primaryEntityId, aliasEntityId].sort().join(':');
+    if (seen.has(key)) return;
+    seen.add(key);
+    aliases.push({ primaryEntityId, aliasEntityId, confidence, matchReason });
+  };
+
+  // Tier 1: Email match (confidence 1.0)
+  const emailGroups = new Map<string, IdentityCandidate[]>();
+  for (const entity of entities) {
+    const email = entity.properties['email'];
+    if (typeof email !== 'string' || email.trim() === '') continue;
+    const normalizedEmail = email.toLowerCase();
+    const group = emailGroups.get(normalizedEmail) ?? [];
+    group.push(entity);
+    emailGroups.set(normalizedEmail, group);
+  }
+
+  for (const group of emailGroups.values()) {
+    if (group.length < 2) continue;
+    const primary = group[0]!;
+    for (let i = 1; i < group.length; i++) {
+      addAlias(primary.id, group[i]!.id, 1.0, 'email_match');
+    }
+  }
+
+  // Tier 2: Fuzzy name match (confidence 0.6–0.8)
+  // Threshold: Jaro-Winkler >= 0.85, same entity type required
+  const JARO_WINKLER_THRESHOLD = 0.85;
+
+  for (let i = 0; i < entities.length; i++) {
+    for (let j = i + 1; j < entities.length; j++) {
+      const a = entities[i]!;
+      const b = entities[j]!;
+
+      // Must be same entity type
+      if (a.entityType !== b.entityType) continue;
+
+      // Skip if already aliased
+      const key = [a.id, b.id].sort().join(':');
+      if (seen.has(key)) continue;
+
+      const normA = normalizeName(a.canonicalName);
+      const normB = normalizeName(b.canonicalName);
+      const similarity = jaroWinkler(normA, normB);
+
+      if (similarity >= JARO_WINKLER_THRESHOLD) {
+        // Map similarity [0.85, 1.0] → confidence [0.6, 0.8]
+        const confidence = 0.6 + ((similarity - 0.85) / 0.15) * 0.2;
+        const clampedConfidence = Math.min(0.8, Math.max(0.6, confidence));
+        addAlias(a.id, b.id, clampedConfidence, 'fuzzy_name');
+      }
+    }
+  }
+
+  return aliases;
+}

package/src/index.ts

@@ -10,3 +10,4 @@ export * from './sessions/index.js';
 export * from './feedback/index.js';
 export * from './profiles/index.js';
 export * from './scratchpad/index.js';
+export * from './identity/index.js';

package/src/models/entity.ts

@@ -16,6 +16,19 @@ export const EntitySchema = z.object({
 
 export type Entity = z.infer<typeof EntitySchema>;
 
+export const AliasSchema = z.object({
+  id: z.string().uuid(),
+  tenantId: z.string().uuid(),
+  primaryEntityId: z.string().uuid(),
+  aliasEntityId: z.string().uuid(),
+  confidence: z.number().min(0).max(1),
+  matchReason: z.enum(['email_match', 'fuzzy_name', 'temporal_cooccurrence', 'manual']),
+  confirmedBy: z.string().nullable(),
+  confirmedAt: z.coerce.date().nullable(),
+  createdAt: z.coerce.date(),
+});
+export type Alias = z.infer<typeof AliasSchema>;
+
 export const CreateEntitySchema = z.object({
   tenantId: z.string().uuid(),
   name: z.string().min(1).max(500),

package/src/retrieval/graph-traversal.ts

@@ -41,14 +41,17 @@ export function tokenizeQuery(query: string): string[] {
  *    - 1-hop = 0.5
  *    - 2-hop = 0.25
  *    - 3-hop = 0.125
+ *
+ * Note: Entity lookups are tenant-scoped.
+ * Fact retrieval is scope-filtered to prevent cross-user data leakage.
  */
 export async function graphSearch(
   storage: StorageAdapter,
   embedding: EmbeddingAdapter,
   query: string,
   tenantId: string,
-  _scope: string,
-  _scopeId: string,
+  scope: string,
+  scopeId: string,
   limit: number,
   config?: Partial<GraphSearchConfig>,
 ): Promise<Candidate[]> {
@@ -183,7 +186,7 @@ export async function graphSearch(
     // Single query: get all facts linked to any of these entities
     try {
       const batchResult = await storage.getFactsForEntities(
-        tenantId, entityIds, PER_ENTITY_LIMIT
+        tenantId, entityIds, PER_ENTITY_LIMIT, scope, scopeId
       );
 
       for (const { entityId, fact } of batchResult) {
@@ -214,7 +217,7 @@ export async function graphSearch(
         const hopDepth = entityHopMap.get(entity.id) ?? maxDepth;
         const graphScore = 1 / Math.pow(2, hopDepth);
         try {
-          const factsResult = await storage.getFactsForEntity(tenantId, entity.id, { limit: 3 });
+          const factsResult = await storage.getFactsForEntity(tenantId, entity.id, { limit: 3 }, scope, scopeId);
           for (const fact of factsResult.data) {
             if (!candidateMap.has(fact.id)) {
               candidateMap.set(fact.id, {