audrey 0.14.0 → 0.16.0

package/src/embedding.js

@@ -1,256 +1,256 @@
-import { createHash } from 'node:crypto';
-
-/**
- * @typedef {Object} EmbeddingProvider
- * @property {number} dimensions
- * @property {string} modelName
- * @property {string} modelVersion
- * @property {(text: string) => Promise<number[]>} embed
- * @property {(texts: string[]) => Promise<number[][]>} embedBatch
- * @property {(vector: number[]) => Buffer} vectorToBuffer
- * @property {(buffer: Buffer) => number[]} bufferToVector
- */
-
-/** @implements {EmbeddingProvider} */
-export class MockEmbeddingProvider {
-  constructor({ dimensions = 64 } = {}) {
-    this.dimensions = dimensions;
-    this.modelName = 'mock-embedding';
-    this.modelVersion = '1.0.0';
-  }
-
-  async embed(text) {
-    const hash = createHash('sha256').update(text).digest();
-    const vector = new Array(this.dimensions);
-    for (let i = 0; i < this.dimensions; i++) {
-      vector[i] = (hash[i % hash.length] / 255) * 2 - 1;
-    }
-    const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
-    return vector.map(v => v / magnitude);
-  }
-
-  async embedBatch(texts) {
-    return Promise.all(texts.map(t => this.embed(t)));
-  }
-
-  vectorToBuffer(vector) {
-    return Buffer.from(new Float32Array(vector).buffer);
-  }
-
-  bufferToVector(buffer) {
-    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
-  }
-}
-
-/** @implements {EmbeddingProvider} */
-export class OpenAIEmbeddingProvider {
-  constructor({ apiKey, model = 'text-embedding-3-small', dimensions = 1536, timeout = 30000 } = {}) {
-    this.apiKey = apiKey || process.env.OPENAI_API_KEY;
-    this.model = model;
-    this.dimensions = dimensions;
-    this.timeout = timeout;
-    this.modelName = model;
-    this.modelVersion = 'latest';
-  }
-
-  async embed(text) {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.timeout);
-    try {
-      const response = await fetch('https://api.openai.com/v1/embeddings', {
-        method: 'POST',
-        headers: {
-          'Authorization': `Bearer ${this.apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({ input: text, model: this.model, dimensions: this.dimensions }),
-        signal: controller.signal,
-      });
-      if (!response.ok) throw new Error(`OpenAI embedding failed: ${response.status}`);
-      const data = await response.json();
-      return data.data[0].embedding;
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-
-  async embedBatch(texts) {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.timeout);
-    try {
-      const response = await fetch('https://api.openai.com/v1/embeddings', {
-        method: 'POST',
-        headers: {
-          'Authorization': `Bearer ${this.apiKey}`,
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({ input: texts, model: this.model, dimensions: this.dimensions }),
-        signal: controller.signal,
-      });
-      if (!response.ok) throw new Error(`OpenAI embedding failed: ${response.status}`);
-      const data = await response.json();
-      return data.data.map(d => d.embedding);
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-
-  vectorToBuffer(vector) {
-    return Buffer.from(new Float32Array(vector).buffer);
-  }
-
-  bufferToVector(buffer) {
-    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
-  }
-}
-
-/** @implements {EmbeddingProvider} */
-export class LocalEmbeddingProvider {
-  constructor({ model = 'Xenova/all-MiniLM-L6-v2', device = 'gpu', batchSize = 64 } = {}) {
-    this.model = model;
-    this.dimensions = 384;
-    this.modelName = model;
-    this.modelVersion = '1.0.0';
-    this.device = device;
-    this.batchSize = batchSize;
-    this._pipeline = null;
-    this._readyPromise = null;
-    this._actualDevice = null;
-  }
-
-  ready() {
-    if (!this._readyPromise) {
-      this._readyPromise = (async () => {
-        const { pipeline } = await import('@huggingface/transformers');
-        try {
-          this._pipeline = await pipeline('feature-extraction', this.model, {
-            dtype: 'fp32', device: this.device,
-          });
-          this._actualDevice = this.device;
-        } catch {
-          this._pipeline = await pipeline('feature-extraction', this.model, {
-            dtype: 'fp32', device: 'cpu',
-          });
-          this._actualDevice = 'cpu';
-        }
-      })();
-    }
-    return this._readyPromise;
-  }
-
-  async embed(text) {
-    await this.ready();
-    const output = await this._pipeline(text, { pooling: 'mean', normalize: true });
-    return Array.from(output.data);
-  }
-
-  async embedBatch(texts) {
-    if (texts.length === 0) return [];
-    await this.ready();
-    const results = [];
-    for (let i = 0; i < texts.length; i += this.batchSize) {
-      const chunk = texts.slice(i, i + this.batchSize);
-      const output = await this._pipeline(chunk, { pooling: 'mean', normalize: true });
-      results.push(...output.tolist());
-    }
-    return results;
-  }
-
-  vectorToBuffer(vector) {
-    return Buffer.from(new Float32Array(vector).buffer);
-  }
-
-  bufferToVector(buffer) {
-    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
-  }
-}
-
-/** @implements {EmbeddingProvider} */
-export class GeminiEmbeddingProvider {
-  constructor({ apiKey, model = 'gemini-embedding-001', timeout = 30000 } = {}) {
-    this.apiKey = apiKey || process.env.GOOGLE_API_KEY;
-    this.model = model;
-    this.dimensions = 3072;
-    this.timeout = timeout;
-    this.modelName = model;
-    this.modelVersion = 'latest';
-  }
-
-  async embed(text) {
-    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.timeout);
-    try {
-      const response = await fetch(
-        `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent?key=${this.apiKey}`,
-        {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ model: `models/${this.model}`, content: { parts: [{ text }] } }),
-          signal: controller.signal,
-        }
-      );
-      if (!response.ok) throw new Error(`Gemini embedding failed: ${response.status}`);
-      const data = await response.json();
-      return data.embedding.values;
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-
-  async embedBatch(texts) {
-    if (texts.length === 0) return [];
-    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
-    const results = [];
-    for (let i = 0; i < texts.length; i += 100) {
-      const chunk = texts.slice(i, i + 100);
-      const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(), this.timeout);
-      try {
-        const response = await fetch(
-          `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents?key=${this.apiKey}`,
-          {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({
-              requests: chunk.map(text => ({
-                model: `models/${this.model}`,
-                content: { parts: [{ text }] },
-              })),
-            }),
-            signal: controller.signal,
-          }
-        );
-        if (!response.ok) throw new Error(`Gemini batch embedding failed: ${response.status}`);
-        const data = await response.json();
-        results.push(...data.embeddings.map(e => e.values));
-      } finally {
-        clearTimeout(timer);
-      }
-    }
-    return results;
-  }
-
-  vectorToBuffer(vector) {
-    return Buffer.from(new Float32Array(vector).buffer);
-  }
-
-  bufferToVector(buffer) {
-    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
-  }
-}
-
-export function createEmbeddingProvider(config) {
-  switch (config.provider) {
-    case 'mock':
-      return new MockEmbeddingProvider(config);
-    case 'openai':
-      return new OpenAIEmbeddingProvider(config);
-    case 'local':
-      return new LocalEmbeddingProvider(config);
-    case 'gemini':
-      return new GeminiEmbeddingProvider(config);
-    default:
-      throw new Error(`Unknown embedding provider: ${config.provider}. Valid: mock, openai, local, gemini`);
-  }
-}
+import { createHash } from 'node:crypto';
+
+/**
+ * @typedef {Object} EmbeddingProvider
+ * @property {number} dimensions
+ * @property {string} modelName
+ * @property {string} modelVersion
+ * @property {(text: string) => Promise<number[]>} embed
+ * @property {(texts: string[]) => Promise<number[][]>} embedBatch
+ * @property {(vector: number[]) => Buffer} vectorToBuffer
+ * @property {(buffer: Buffer) => number[]} bufferToVector
+ */
+
+/** @implements {EmbeddingProvider} */
+export class MockEmbeddingProvider {
+  constructor({ dimensions = 64 } = {}) {
+    this.dimensions = dimensions;
+    this.modelName = 'mock-embedding';
+    this.modelVersion = '1.0.0';
+  }
+
+  async embed(text) {
+    const hash = createHash('sha256').update(text).digest();
+    const vector = new Array(this.dimensions);
+    for (let i = 0; i < this.dimensions; i++) {
+      vector[i] = (hash[i % hash.length] / 255) * 2 - 1;
+    }
+    const magnitude = Math.sqrt(vector.reduce((sum, v) => sum + v * v, 0));
+    return vector.map(v => v / magnitude);
+  }
+
+  async embedBatch(texts) {
+    return Promise.all(texts.map(t => this.embed(t)));
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
+/** @implements {EmbeddingProvider} */
+export class OpenAIEmbeddingProvider {
+  constructor({ apiKey, model = 'text-embedding-3-small', dimensions = 1536, timeout = 30000 } = {}) {
+    this.apiKey = apiKey || process.env.OPENAI_API_KEY;
+    this.model = model;
+    this.dimensions = dimensions;
+    this.timeout = timeout;
+    this.modelName = model;
+    this.modelVersion = 'latest';
+  }
+
+  async embed(text) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch('https://api.openai.com/v1/embeddings', {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ input: text, model: this.model, dimensions: this.dimensions }),
+        signal: controller.signal,
+      });
+      if (!response.ok) throw new Error(`OpenAI embedding failed: ${response.status}`);
+      const data = await response.json();
+      return data.data[0].embedding;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async embedBatch(texts) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch('https://api.openai.com/v1/embeddings', {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ input: texts, model: this.model, dimensions: this.dimensions }),
+        signal: controller.signal,
+      });
+      if (!response.ok) throw new Error(`OpenAI embedding failed: ${response.status}`);
+      const data = await response.json();
+      return data.data.map(d => d.embedding);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
+/** @implements {EmbeddingProvider} */
+export class LocalEmbeddingProvider {
+  constructor({ model = 'Xenova/all-MiniLM-L6-v2', device = 'gpu', batchSize = 64 } = {}) {
+    this.model = model;
+    this.dimensions = 384;
+    this.modelName = model;
+    this.modelVersion = '1.0.0';
+    this.device = device;
+    this.batchSize = batchSize;
+    this._pipeline = null;
+    this._readyPromise = null;
+    this._actualDevice = null;
+  }
+
+  ready() {
+    if (!this._readyPromise) {
+      this._readyPromise = (async () => {
+        const { pipeline } = await import('@huggingface/transformers');
+        try {
+          this._pipeline = await pipeline('feature-extraction', this.model, {
+            dtype: 'fp32', device: this.device,
+          });
+          this._actualDevice = this.device;
+        } catch {
+          this._pipeline = await pipeline('feature-extraction', this.model, {
+            dtype: 'fp32', device: 'cpu',
+          });
+          this._actualDevice = 'cpu';
+        }
+      })();
+    }
+    return this._readyPromise;
+  }
+
+  async embed(text) {
+    await this.ready();
+    const output = await this._pipeline(text, { pooling: 'mean', normalize: true });
+    return Array.from(output.data);
+  }
+
+  async embedBatch(texts) {
+    if (texts.length === 0) return [];
+    await this.ready();
+    const results = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const chunk = texts.slice(i, i + this.batchSize);
+      const output = await this._pipeline(chunk, { pooling: 'mean', normalize: true });
+      results.push(...output.tolist());
+    }
+    return results;
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
+/** @implements {EmbeddingProvider} */
+export class GeminiEmbeddingProvider {
+  constructor({ apiKey, model = 'gemini-embedding-001', timeout = 30000 } = {}) {
+    this.apiKey = apiKey || process.env.GOOGLE_API_KEY;
+    this.model = model;
+    this.dimensions = 3072;
+    this.timeout = timeout;
+    this.modelName = model;
+    this.modelVersion = 'latest';
+  }
+
+  async embed(text) {
+    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch(
+        `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent?key=${this.apiKey}`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ model: `models/${this.model}`, content: { parts: [{ text }] } }),
+          signal: controller.signal,
+        }
+      );
+      if (!response.ok) throw new Error(`Gemini embedding failed: ${response.status}`);
+      const data = await response.json();
+      return data.embedding.values;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async embedBatch(texts) {
+    if (texts.length === 0) return [];
+    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
+    const results = [];
+    for (let i = 0; i < texts.length; i += 100) {
+      const chunk = texts.slice(i, i + 100);
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), this.timeout);
+      try {
+        const response = await fetch(
+          `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents?key=${this.apiKey}`,
+          {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              requests: chunk.map(text => ({
+                model: `models/${this.model}`,
+                content: { parts: [{ text }] },
+              })),
+            }),
+            signal: controller.signal,
+          }
+        );
+        if (!response.ok) throw new Error(`Gemini batch embedding failed: ${response.status}`);
+        const data = await response.json();
+        results.push(...data.embeddings.map(e => e.values));
+      } finally {
+        clearTimeout(timer);
+      }
+    }
+    return results;
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
+export function createEmbeddingProvider(config) {
+  switch (config.provider) {
+    case 'mock':
+      return new MockEmbeddingProvider(config);
+    case 'openai':
+      return new OpenAIEmbeddingProvider(config);
+    case 'local':
+      return new LocalEmbeddingProvider(config);
+    case 'gemini':
+      return new GeminiEmbeddingProvider(config);
+    default:
+      throw new Error(`Unknown embedding provider: ${config.provider}. Valid: mock, openai, local, gemini`);
+  }
+}

package/src/export.js

@@ -1,61 +1,67 @@
-import { readFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
-import { join, dirname } from 'node:path';
-import { safeJsonParse } from './utils.js';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-const pkg = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8'));
-
-export function exportMemories(db) {
-  const episodes = db.prepare(
-    'SELECT id, content, source, source_reliability, salience, context, affect, tags, causal_trigger, causal_consequence, created_at, supersedes, superseded_by, consolidated, "private" FROM episodes'
-  ).all().map(ep => ({
-    ...ep,
-    tags: safeJsonParse(ep.tags, null),
-    context: safeJsonParse(ep.context, null),
-    affect: safeJsonParse(ep.affect, null),
-  }));
-
-  const semantics = db.prepare(
-    'SELECT id, content, state, conditions, evidence_episode_ids, evidence_count, supporting_count, contradicting_count, source_type_diversity, consolidation_checkpoint, created_at, last_reinforced_at, retrieval_count, challenge_count, interference_count, salience FROM semantics'
-  ).all().map(sem => ({
-    ...sem,
-    evidence_episode_ids: safeJsonParse(sem.evidence_episode_ids, []),
-  }));
-
-  const procedures = db.prepare(
-    'SELECT id, content, state, trigger_conditions, evidence_episode_ids, success_count, failure_count, created_at, last_reinforced_at, retrieval_count, interference_count, salience FROM procedures'
-  ).all().map(proc => ({
-    ...proc,
-    evidence_episode_ids: safeJsonParse(proc.evidence_episode_ids, []),
-  }));
-
-  const causalLinks = db.prepare('SELECT * FROM causal_links').all();
-
-  const contradictions = db.prepare(
-    'SELECT id, claim_a_id, claim_a_type, claim_b_id, claim_b_type, state, resolution, resolved_at, reopened_at, reopen_evidence_id, created_at FROM contradictions'
-  ).all();
-
-  const consolidationRuns = db.prepare(
-    'SELECT id, input_episode_ids, output_memory_ids, started_at, completed_at, status FROM consolidation_runs'
-  ).all().map(run => ({
-    ...run,
-    input_episode_ids: safeJsonParse(run.input_episode_ids, []),
-    output_memory_ids: safeJsonParse(run.output_memory_ids, []),
-  }));
-
-  const configRows = db.prepare('SELECT key, value FROM audrey_config').all();
-  const config = Object.fromEntries(configRows.map(r => [r.key, r.value]));
-
-  return {
-    version: pkg.version,
-    exportedAt: new Date().toISOString(),
-    episodes,
-    semantics,
-    procedures,
-    causalLinks,
-    contradictions,
-    consolidationRuns,
-    config,
-  };
-}
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { join, dirname } from 'node:path';
+import { safeJsonParse } from './utils.js';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const pkg = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8'));
+
+export function exportMemories(db) {
+  const episodes = db.prepare(
+    'SELECT id, content, source, source_reliability, salience, context, affect, tags, causal_trigger, causal_consequence, created_at, embedding_model, embedding_version, supersedes, superseded_by, consolidated, "private" FROM episodes'
+  ).all().map(ep => ({
+    ...ep,
+    tags: safeJsonParse(ep.tags, null),
+    context: safeJsonParse(ep.context, null),
+    affect: safeJsonParse(ep.affect, null),
+  }));
+
+  const semantics = db.prepare(
+    'SELECT id, content, state, conditions, evidence_episode_ids, evidence_count, supporting_count, contradicting_count, source_type_diversity, consolidation_checkpoint, embedding_model, embedding_version, consolidation_model, consolidation_prompt_hash, created_at, last_reinforced_at, retrieval_count, challenge_count, interference_count, salience FROM semantics'
+  ).all().map(sem => ({
+    ...sem,
+    evidence_episode_ids: safeJsonParse(sem.evidence_episode_ids, []),
+  }));
+
+  const procedures = db.prepare(
+    'SELECT id, content, state, trigger_conditions, evidence_episode_ids, success_count, failure_count, embedding_model, embedding_version, created_at, last_reinforced_at, retrieval_count, interference_count, salience FROM procedures'
+  ).all().map(proc => ({
+    ...proc,
+    evidence_episode_ids: safeJsonParse(proc.evidence_episode_ids, []),
+  }));
+
+  const causalLinks = db.prepare('SELECT * FROM causal_links').all();
+
+  const contradictions = db.prepare(
+    'SELECT id, claim_a_id, claim_a_type, claim_b_id, claim_b_type, state, resolution, resolved_at, reopened_at, reopen_evidence_id, created_at FROM contradictions'
+  ).all();
+
+  const consolidationRuns = db.prepare(
+    'SELECT id, checkpoint_cursor, input_episode_ids, output_memory_ids, confidence_deltas, consolidation_model, consolidation_prompt_hash, started_at, completed_at, status FROM consolidation_runs'
+  ).all().map(run => ({
+    ...run,
+    confidence_deltas: safeJsonParse(run.confidence_deltas, null),
+    input_episode_ids: safeJsonParse(run.input_episode_ids, []),
+    output_memory_ids: safeJsonParse(run.output_memory_ids, []),
+  }));
+
+  const consolidationMetrics = db.prepare(
+    'SELECT id, run_id, min_cluster_size, similarity_threshold, episodes_evaluated, clusters_found, principles_extracted, created_at FROM consolidation_metrics'
+  ).all();
+
+  const configRows = db.prepare('SELECT key, value FROM audrey_config').all();
+  const config = Object.fromEntries(configRows.map(r => [r.key, r.value]));
+
+  return {
+    version: pkg.version,
+    exportedAt: new Date().toISOString(),
+    episodes,
+    semantics,
+    procedures,
+    causalLinks,
+    contradictions,
+    consolidationRuns,
+    consolidationMetrics,
+    config,
+  };
+}