@soleri/core 2.8.0 → 2.10.0

package/src/streams/replayable-stream.ts

@@ -0,0 +1,92 @@
+/**
+ * Multi-consumer async stream that replays from a buffer.
+ * Source executes exactly once — new iterators replay from buffer[0].
+ */
+export class ReplayableStream<T> implements AsyncIterable<T> {
+  private buffer: T[] = [];
+  private source: AsyncIterator<T>;
+  private done = false;
+  private error: unknown = undefined;
+  private waiters: Array<{
+    resolve: (result: IteratorResult<T, undefined>) => void;
+    reject: (err: unknown) => void;
+  }> = [];
+  private advancing = false;
+
+  constructor(source: AsyncIterable<T>) {
+    this.source = source[Symbol.asyncIterator]();
+  }
+
+  private async advance(): Promise<void> {
+    if (this.advancing || this.done) return;
+    this.advancing = true;
+    try {
+      const result = await this.source.next();
+      if (result.done) {
+        this.done = true;
+        for (const waiter of this.waiters) {
+          waiter.resolve({ value: undefined, done: true });
+        }
+        this.waiters.length = 0;
+      } else {
+        this.buffer.push(result.value);
+        for (const waiter of this.waiters) {
+          waiter.resolve({ value: result.value, done: false });
+        }
+        this.waiters.length = 0;
+      }
+    } catch (err) {
+      this.done = true;
+      this.error = err;
+      for (const waiter of this.waiters) {
+        waiter.reject(err);
+      }
+      this.waiters.length = 0;
+    } finally {
+      this.advancing = false;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    let index = 0;
+    return {
+      next: async (): Promise<IteratorResult<T>> => {
+        if (index < this.buffer.length) {
+          return { value: this.buffer[index++], done: false };
+        }
+        if (this.done) {
+          if (this.error) throw this.error;
+          return { value: undefined, done: true };
+        }
+        // Need to advance the source
+        return new Promise<IteratorResult<T, undefined>>((resolve, reject) => {
+          this.waiters.push({ resolve, reject });
+          if (!this.advancing) {
+            this.advance().catch(() => {
+              // Error already propagated to waiters via reject
+            });
+          }
+        }).then((result) => {
+          if (!result.done) index++;
+          return result;
+        });
+      },
+    };
+  }
+
+  async collect(): Promise<T[]> {
+    const items: T[] = [];
+    for await (const item of this) {
+      items.push(item);
+    }
+    return items;
+  }
+
+  get bufferedCount(): number {
+    return this.buffer.length;
+  }
+
+  get isDone(): boolean {
+    return this.done;
+  }
+}

package/src/vault/content-hash.ts

@@ -0,0 +1,31 @@
+import { createHash } from 'node:crypto';
+
+export interface HashableEntry {
+  type: string;
+  domain: string;
+  title: string;
+  description: string;
+  tags?: string[];
+  example?: string;
+  counterExample?: string;
+}
+
+/**
+ * Compute a deterministic SHA-256 content hash for a vault entry.
+ * Normalizes fields (lowercase domain, trim, sort tags/keys) before hashing.
+ * Returns 40-char hex string. Excludes mutable fields (id, severity, timestamps).
+ */
+export function computeContentHash(entry: HashableEntry): string {
+  const normalized = {
+    counterExample: (entry.counterExample ?? '').trim(),
+    description: entry.description.trim(),
+    domain: entry.domain.toLowerCase().trim(),
+    example: (entry.example ?? '').trim(),
+    tags: [...(entry.tags ?? [])].sort(),
+    title: entry.title.trim(),
+    type: entry.type.trim(),
+  };
+  // Keys already alphabetical — JSON.stringify preserves insertion order
+  const json = JSON.stringify(normalized);
+  return createHash('sha256').update(json, 'utf8').digest('hex').slice(0, 40);
+}

package/src/vault/vault.ts

@@ -1,6 +1,7 @@
 import type { PersistenceProvider } from '../persistence/types.js';
 import { SQLitePersistenceProvider } from '../persistence/sqlite-provider.js';
 import type { IntelligenceEntry } from '../intelligence/types.js';
+import { computeContentHash } from './content-hash.js';
 
 export interface SearchResult {
   entry: IntelligenceEntry;
@@ -177,6 +178,7 @@ export class Vault {
     `);
     this.migrateBrainSchema();
     this.migrateTemporalSchema();
+    this.migrateContentHash();
   }
 
   private migrateTemporalSchema(): void {
@@ -192,6 +194,49 @@ export class Vault {
     }
   }
 
+  private migrateContentHash(): void {
+    try {
+      this.provider.run('ALTER TABLE entries ADD COLUMN content_hash TEXT');
+    } catch {
+      // Column already exists
+    }
+    this.provider.execSql(
+      'CREATE INDEX IF NOT EXISTS idx_entries_content_hash ON entries(content_hash) WHERE content_hash IS NOT NULL',
+    );
+    // Backfill existing entries that lack a hash
+    const unhashed = this.provider.all<{
+      id: string;
+      type: string;
+      domain: string;
+      title: string;
+      description: string;
+      tags: string;
+      example: string | null;
+      counter_example: string | null;
+    }>(
+      'SELECT id, type, domain, title, description, tags, example, counter_example FROM entries WHERE content_hash IS NULL',
+    );
+    if (unhashed.length > 0) {
+      this.provider.transaction(() => {
+        for (const row of unhashed) {
+          const hash = computeContentHash({
+            type: row.type,
+            domain: row.domain,
+            title: row.title,
+            description: row.description,
+            tags: JSON.parse(row.tags),
+            example: row.example ?? undefined,
+            counterExample: row.counter_example ?? undefined,
+          });
+          this.provider.run('UPDATE entries SET content_hash = @hash WHERE id = @id', {
+            hash,
+            id: row.id,
+          });
+        }
+      });
+    }
+  }
+
   private migrateBrainSchema(): void {
     const columns = this.provider.all<{ name: string }>('PRAGMA table_info(brain_feedback)');
     const hasSource = columns.some((c) => c.name === 'source');
@@ -236,11 +281,12 @@ export class Vault {
 
   seed(entries: IntelligenceEntry[]): number {
     const sql = `
-      INSERT INTO entries (id,type,domain,title,severity,description,context,example,counter_example,why,tags,applies_to,valid_from,valid_until)
-      VALUES (@id,@type,@domain,@title,@severity,@description,@context,@example,@counterExample,@why,@tags,@appliesTo,@validFrom,@validUntil)
+      INSERT INTO entries (id,type,domain,title,severity,description,context,example,counter_example,why,tags,applies_to,valid_from,valid_until,content_hash)
+      VALUES (@id,@type,@domain,@title,@severity,@description,@context,@example,@counterExample,@why,@tags,@appliesTo,@validFrom,@validUntil,@contentHash)
       ON CONFLICT(id) DO UPDATE SET type=excluded.type,domain=excluded.domain,title=excluded.title,severity=excluded.severity,
         description=excluded.description,context=excluded.context,example=excluded.example,counter_example=excluded.counter_example,
-        why=excluded.why,tags=excluded.tags,applies_to=excluded.applies_to,valid_from=excluded.valid_from,valid_until=excluded.valid_until,updated_at=unixepoch()
+        why=excluded.why,tags=excluded.tags,applies_to=excluded.applies_to,valid_from=excluded.valid_from,valid_until=excluded.valid_until,
+        content_hash=excluded.content_hash,updated_at=unixepoch()
     `;
     return this.provider.transaction(() => {
       let count = 0;
@@ -260,6 +306,7 @@ export class Vault {
           appliesTo: JSON.stringify(entry.appliesTo ?? []),
           validFrom: entry.validFrom ?? null,
           validUntil: entry.validUntil ?? null,
+          contentHash: computeContentHash(entry),
         });
         count++;
         if (this.syncManager) {
@@ -999,6 +1046,29 @@ export class Vault {
     throw new Error('getDb() is only available with SQLite provider');
   }
 
+  /** Check if an entry with this content hash already exists. Returns the existing ID or null. */
+  findByContentHash(hash: string): string | null {
+    const row = this.provider.get<{ id: string }>(
+      'SELECT id FROM entries WHERE content_hash = @hash',
+      { hash },
+    );
+    return row?.id ?? null;
+  }
+
+  /** Get content hash stats for dedup reporting. */
+  contentHashStats(): { total: number; hashed: number; uniqueHashes: number } {
+    const total = this.provider.get<{ c: number }>('SELECT COUNT(*) as c FROM entries')?.c ?? 0;
+    const hashed =
+      this.provider.get<{ c: number }>(
+        'SELECT COUNT(*) as c FROM entries WHERE content_hash IS NOT NULL',
+      )?.c ?? 0;
+    const uniqueHashes =
+      this.provider.get<{ c: number }>(
+        'SELECT COUNT(DISTINCT content_hash) as c FROM entries WHERE content_hash IS NOT NULL',
+      )?.c ?? 0;
+    return { total, hashed, uniqueHashes };
+  }
+
   close(): void {
     this.provider.close();
   }