audrey 0.9.0 → 0.14.0

package/src/embedding.js

@@ -11,33 +11,14 @@ import { createHash } from 'node:crypto';
  * @property {(buffer: Buffer) => number[]} bufferToVector
  */
 
-/**
- * @typedef {Object} MockEmbeddingConfig
- * @property {'mock'} provider
- * @property {number} [dimensions=64]
- */
-
-/**
- * @typedef {Object} OpenAIEmbeddingConfig
- * @property {'openai'} provider
- * @property {string} [apiKey]
- * @property {string} [model='text-embedding-3-small']
- * @property {number} [dimensions=1536]
- */
-
 /** @implements {EmbeddingProvider} */
 export class MockEmbeddingProvider {
-  /** @param {Partial<MockEmbeddingConfig>} [config={}] */
   constructor({ dimensions = 64 } = {}) {
     this.dimensions = dimensions;
     this.modelName = 'mock-embedding';
     this.modelVersion = '1.0.0';
   }
 
-  /**
-   * @param {string} text
-   * @returns {Promise<number[]>}
-   */
   async embed(text) {
     const hash = createHash('sha256').update(text).digest();
     const vector = new Array(this.dimensions);
@@ -48,26 +29,14 @@ export class MockEmbeddingProvider {
     return vector.map(v => v / magnitude);
   }
 
-  /**
-   * @param {string[]} texts
-   * @returns {Promise<number[][]>}
-   */
   async embedBatch(texts) {
     return Promise.all(texts.map(t => this.embed(t)));
   }
 
-  /**
-   * @param {number[]} vector
-   * @returns {Buffer}
-   */
   vectorToBuffer(vector) {
     return Buffer.from(new Float32Array(vector).buffer);
   }
 
-  /**
-   * @param {Buffer} buffer
-   * @returns {number[]}
-   */
   bufferToVector(buffer) {
     return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
   }
@@ -75,7 +44,6 @@ export class MockEmbeddingProvider {
 
 /** @implements {EmbeddingProvider} */
 export class OpenAIEmbeddingProvider {
-  /** @param {Partial<OpenAIEmbeddingConfig>} [config={}] */
   constructor({ apiKey, model = 'text-embedding-3-small', dimensions = 1536, timeout = 30000 } = {}) {
     this.apiKey = apiKey || process.env.OPENAI_API_KEY;
     this.model = model;
@@ -85,10 +53,6 @@ export class OpenAIEmbeddingProvider {
     this.modelVersion = 'latest';
   }
 
-  /**
-   * @param {string} text
-   * @returns {Promise<number[]>}
-   */
   async embed(text) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.timeout);
@@ -110,10 +74,6 @@ export class OpenAIEmbeddingProvider {
     }
   }
 
-  /**
-   * @param {string[]} texts
-   * @returns {Promise<number[][]>}
-   */
   async embedBatch(texts) {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.timeout);
@@ -135,34 +95,162 @@ export class OpenAIEmbeddingProvider {
     }
   }
 
-  /**
-   * @param {number[]} vector
-   * @returns {Buffer}
-   */
   vectorToBuffer(vector) {
     return Buffer.from(new Float32Array(vector).buffer);
   }
 
-  /**
-   * @param {Buffer} buffer
-   * @returns {number[]}
-   */
   bufferToVector(buffer) {
     return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
   }
 }
 
-/**
- * @param {MockEmbeddingConfig | OpenAIEmbeddingConfig} config
- * @returns {MockEmbeddingProvider | OpenAIEmbeddingProvider}
- */
+/** @implements {EmbeddingProvider} */
+export class LocalEmbeddingProvider {
+  constructor({ model = 'Xenova/all-MiniLM-L6-v2', device = 'gpu', batchSize = 64 } = {}) {
+    this.model = model;
+    this.dimensions = 384;
+    this.modelName = model;
+    this.modelVersion = '1.0.0';
+    this.device = device;
+    this.batchSize = batchSize;
+    this._pipeline = null;
+    this._readyPromise = null;
+    this._actualDevice = null;
+  }
+
+  ready() {
+    if (!this._readyPromise) {
+      this._readyPromise = (async () => {
+        const { pipeline } = await import('@huggingface/transformers');
+        try {
+          this._pipeline = await pipeline('feature-extraction', this.model, {
+            dtype: 'fp32', device: this.device,
+          });
+          this._actualDevice = this.device;
+        } catch {
+          this._pipeline = await pipeline('feature-extraction', this.model, {
+            dtype: 'fp32', device: 'cpu',
+          });
+          this._actualDevice = 'cpu';
+        }
+      })();
+    }
+    return this._readyPromise;
+  }
+
+  async embed(text) {
+    await this.ready();
+    const output = await this._pipeline(text, { pooling: 'mean', normalize: true });
+    return Array.from(output.data);
+  }
+
+  async embedBatch(texts) {
+    if (texts.length === 0) return [];
+    await this.ready();
+    const results = [];
+    for (let i = 0; i < texts.length; i += this.batchSize) {
+      const chunk = texts.slice(i, i + this.batchSize);
+      const output = await this._pipeline(chunk, { pooling: 'mean', normalize: true });
+      results.push(...output.tolist());
+    }
+    return results;
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
+/** @implements {EmbeddingProvider} */
+export class GeminiEmbeddingProvider {
+  constructor({ apiKey, model = 'gemini-embedding-001', timeout = 30000 } = {}) {
+    this.apiKey = apiKey || process.env.GOOGLE_API_KEY;
+    this.model = model;
+    this.dimensions = 3072;
+    this.timeout = timeout;
+    this.modelName = model;
+    this.modelVersion = 'latest';
+  }
+
+  async embed(text) {
+    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), this.timeout);
+    try {
+      const response = await fetch(
+        `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:embedContent?key=${this.apiKey}`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ model: `models/${this.model}`, content: { parts: [{ text }] } }),
+          signal: controller.signal,
+        }
+      );
+      if (!response.ok) throw new Error(`Gemini embedding failed: ${response.status}`);
+      const data = await response.json();
+      return data.embedding.values;
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async embedBatch(texts) {
+    if (texts.length === 0) return [];
+    if (!this.apiKey) throw new Error('Gemini embedding requires GOOGLE_API_KEY');
+    const results = [];
+    for (let i = 0; i < texts.length; i += 100) {
+      const chunk = texts.slice(i, i + 100);
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), this.timeout);
+      try {
+        const response = await fetch(
+          `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:batchEmbedContents?key=${this.apiKey}`,
+          {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+              requests: chunk.map(text => ({
+                model: `models/${this.model}`,
+                content: { parts: [{ text }] },
+              })),
+            }),
+            signal: controller.signal,
+          }
+        );
+        if (!response.ok) throw new Error(`Gemini batch embedding failed: ${response.status}`);
+        const data = await response.json();
+        results.push(...data.embeddings.map(e => e.values));
+      } finally {
+        clearTimeout(timer);
+      }
+    }
+    return results;
+  }
+
+  vectorToBuffer(vector) {
+    return Buffer.from(new Float32Array(vector).buffer);
+  }
+
+  bufferToVector(buffer) {
+    return Array.from(new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4));
+  }
+}
+
 export function createEmbeddingProvider(config) {
   switch (config.provider) {
     case 'mock':
       return new MockEmbeddingProvider(config);
     case 'openai':
       return new OpenAIEmbeddingProvider(config);
+    case 'local':
+      return new LocalEmbeddingProvider(config);
+    case 'gemini':
+      return new GeminiEmbeddingProvider(config);
     default:
-      throw new Error(`Unknown embedding provider: ${config.provider}. Valid: mock, openai`);
+      throw new Error(`Unknown embedding provider: ${config.provider}. Valid: mock, openai, local, gemini`);
   }
 }

package/src/encode.js

@@ -1,61 +1,63 @@
-import { generateId } from './ulid.js';
-import { sourceReliability } from './confidence.js';
-import { arousalSalienceBoost } from './affect.js';
-
-/**
- * @param {import('better-sqlite3').Database} db
- * @param {import('./embedding.js').EmbeddingProvider} embeddingProvider
- * @param {{ content: string, source: string, salience?: number, causal?: { trigger?: string, consequence?: string }, tags?: string[], supersedes?: string }} params
- * @returns {Promise<string>}
- */
-export async function encodeEpisode(db, embeddingProvider, {
-  content,
-  source,
-  salience = 0.5,
-  causal,
-  tags,
-  supersedes,
-  context = {},
-  affect = {},
-  arousalWeight = 0.3,
-}) {
-  if (!content || typeof content !== 'string') throw new Error('content must be a non-empty string');
-  if (salience < 0 || salience > 1) throw new Error('salience must be between 0 and 1');
-  if (tags && !Array.isArray(tags)) throw new Error('tags must be an array');
-
-  const reliability = sourceReliability(source);
-  const vector = await embeddingProvider.embed(content);
-  const embeddingBuffer = embeddingProvider.vectorToBuffer(vector);
-  const id = generateId();
-  const now = new Date().toISOString();
-
-  const boost = arousalSalienceBoost(affect.arousal);
-  const effectiveSalience = Math.min(1.0, salience + (boost * arousalWeight));
-
-  const insertAndLink = db.transaction(() => {
-    db.prepare(`
-      INSERT INTO episodes (
-        id, content, embedding, source, source_reliability, salience, context, affect,
-        tags, causal_trigger, causal_consequence, created_at,
-        embedding_model, embedding_version, supersedes
-      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(
-      id, content, embeddingBuffer, source, reliability, effectiveSalience,
-      JSON.stringify(context),
-      JSON.stringify(affect),
-      tags ? JSON.stringify(tags) : null,
-      causal?.trigger || null, causal?.consequence || null,
-      now, embeddingProvider.modelName, embeddingProvider.modelVersion,
-      supersedes || null,
-    );
-    db.prepare(
-      'INSERT INTO vec_episodes(id, embedding, source, consolidated) VALUES (?, ?, ?, ?)'
-    ).run(id, embeddingBuffer, source, BigInt(0));
-    if (supersedes) {
-      db.prepare('UPDATE episodes SET superseded_by = ? WHERE id = ?').run(id, supersedes);
-    }
-  });
-
-  insertAndLink();
-  return id;
-}
+import { generateId } from './ulid.js';
+import { sourceReliability } from './confidence.js';
+import { arousalSalienceBoost } from './affect.js';
+
+/**
+ * @param {import('better-sqlite3').Database} db
+ * @param {import('./embedding.js').EmbeddingProvider} embeddingProvider
+ * @param {{ content: string, source: string, salience?: number, causal?: { trigger?: string, consequence?: string }, tags?: string[], supersedes?: string, context?: object, affect?: object, arousalWeight?: number, private?: boolean }} params
+ * @returns {Promise<string>}
+ */
+export async function encodeEpisode(db, embeddingProvider, {
+  content,
+  source,
+  salience = 0.5,
+  causal,
+  tags,
+  supersedes,
+  context = {},
+  affect = {},
+  arousalWeight = 0.3,
+  private: isPrivate = false,
+}) {
+  if (!content || typeof content !== 'string') throw new Error('content must be a non-empty string');
+  if (salience < 0 || salience > 1) throw new Error('salience must be between 0 and 1');
+  if (tags && !Array.isArray(tags)) throw new Error('tags must be an array');
+
+  const reliability = sourceReliability(source);
+  const vector = await embeddingProvider.embed(content);
+  const embeddingBuffer = embeddingProvider.vectorToBuffer(vector);
+  const id = generateId();
+  const now = new Date().toISOString();
+
+  const boost = arousalSalienceBoost(affect.arousal);
+  const effectiveSalience = Math.min(1.0, salience + (boost * arousalWeight));
+
+  const insertAndLink = db.transaction(() => {
+    db.prepare(`
+      INSERT INTO episodes (
+        id, content, embedding, source, source_reliability, salience, context, affect,
+        tags, causal_trigger, causal_consequence, created_at,
+        embedding_model, embedding_version, supersedes, "private"
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      id, content, embeddingBuffer, source, reliability, effectiveSalience,
+      JSON.stringify(context),
+      JSON.stringify(affect),
+      tags ? JSON.stringify(tags) : null,
+      causal?.trigger || null, causal?.consequence || null,
+      now, embeddingProvider.modelName, embeddingProvider.modelVersion,
+      supersedes || null,
+      isPrivate ? 1 : 0,
+    );
+    db.prepare(
+      'INSERT INTO vec_episodes(id, embedding, source, consolidated) VALUES (?, ?, ?, ?)'
+    ).run(id, embeddingBuffer, source, BigInt(0));
+    if (supersedes) {
+      db.prepare('UPDATE episodes SET superseded_by = ? WHERE id = ?').run(id, supersedes);
+    }
+  });
+
+  insertAndLink();
+  return id;
+}

package/src/export.js

@@ -8,21 +8,23 @@ const pkg = JSON.parse(readFileSync(join(__dirname, '../package.json'), 'utf-8')
 
 export function exportMemories(db) {
   const episodes = db.prepare(
-    'SELECT id, content, source, source_reliability, salience, tags, causal_trigger, causal_consequence, created_at, supersedes, superseded_by, consolidated FROM episodes'
+    'SELECT id, content, source, source_reliability, salience, context, affect, tags, causal_trigger, causal_consequence, created_at, supersedes, superseded_by, consolidated, "private" FROM episodes'
   ).all().map(ep => ({
     ...ep,
     tags: safeJsonParse(ep.tags, null),
+    context: safeJsonParse(ep.context, null),
+    affect: safeJsonParse(ep.affect, null),
   }));
 
   const semantics = db.prepare(
-    'SELECT id, content, state, conditions, evidence_episode_ids, evidence_count, supporting_count, contradicting_count, source_type_diversity, consolidation_checkpoint, created_at, last_reinforced_at, retrieval_count, challenge_count FROM semantics'
+    'SELECT id, content, state, conditions, evidence_episode_ids, evidence_count, supporting_count, contradicting_count, source_type_diversity, consolidation_checkpoint, created_at, last_reinforced_at, retrieval_count, challenge_count, interference_count, salience FROM semantics'
   ).all().map(sem => ({
     ...sem,
     evidence_episode_ids: safeJsonParse(sem.evidence_episode_ids, []),
   }));
 
   const procedures = db.prepare(
-    'SELECT id, content, state, trigger_conditions, evidence_episode_ids, success_count, failure_count, created_at, last_reinforced_at, retrieval_count FROM procedures'
+    'SELECT id, content, state, trigger_conditions, evidence_episode_ids, success_count, failure_count, created_at, last_reinforced_at, retrieval_count, interference_count, salience FROM procedures'
   ).all().map(proc => ({
     ...proc,
     evidence_episode_ids: safeJsonParse(proc.evidence_episode_ids, []),

package/src/import.js

@@ -5,9 +5,9 @@ export async function importMemories(db, embeddingProvider, snapshot) {
   }
 
   const insertEpisode = db.prepare(`
-    INSERT INTO episodes (id, content, source, source_reliability, salience, tags,
-      causal_trigger, causal_consequence, created_at, supersedes, superseded_by, consolidated)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    INSERT INTO episodes (id, content, source, source_reliability, salience, context, affect, tags,
+      causal_trigger, causal_consequence, created_at, supersedes, superseded_by, consolidated, "private")
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
   const insertVecEpisode = db.prepare(
@@ -17,8 +17,9 @@ export async function importMemories(db, embeddingProvider, snapshot) {
   const insertSemantic = db.prepare(`
     INSERT INTO semantics (id, content, state, conditions, evidence_episode_ids,
       evidence_count, supporting_count, contradicting_count, source_type_diversity,
-      consolidation_checkpoint, created_at, last_reinforced_at, retrieval_count, challenge_count)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      consolidation_checkpoint, created_at, last_reinforced_at, retrieval_count, challenge_count,
+      interference_count, salience)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
   const insertVecSemantic = db.prepare(
@@ -27,8 +28,9 @@ export async function importMemories(db, embeddingProvider, snapshot) {
 
   const insertProcedure = db.prepare(`
     INSERT INTO procedures (id, content, state, trigger_conditions, evidence_episode_ids,
-      success_count, failure_count, created_at, last_reinforced_at, retrieval_count)
-    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      success_count, failure_count, created_at, last_reinforced_at, retrieval_count,
+      interference_count, salience)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   `);
 
   const insertVecProcedure = db.prepare(
@@ -53,10 +55,13 @@ export async function importMemories(db, embeddingProvider, snapshot) {
 
   for (const ep of snapshot.episodes) {
     const tags = ep.tags ? JSON.stringify(ep.tags) : null;
+    const context = ep.context ? JSON.stringify(ep.context) : '{}';
+    const affect = ep.affect ? JSON.stringify(ep.affect) : '{}';
     insertEpisode.run(
       ep.id, ep.content, ep.source, ep.source_reliability, ep.salience ?? 0.5,
-      tags, ep.causal_trigger ?? null, ep.causal_consequence ?? null,
+      context, affect, tags, ep.causal_trigger ?? null, ep.causal_consequence ?? null,
       ep.created_at, ep.supersedes ?? null, ep.superseded_by ?? null, ep.consolidated ?? 0,
+      ep.private ?? 0,
     );
 
     const vector = await embeddingProvider.embed(ep.content);
@@ -71,6 +76,7 @@ export async function importMemories(db, embeddingProvider, snapshot) {
       sem.evidence_count ?? 0, sem.supporting_count ?? 0, sem.contradicting_count ?? 0,
       sem.source_type_diversity ?? 0, sem.consolidation_checkpoint ?? null,
       sem.created_at, sem.last_reinforced_at ?? null, sem.retrieval_count ?? 0, sem.challenge_count ?? 0,
+      sem.interference_count ?? 0, sem.salience ?? 0.5,
     );
 
     const vector = await embeddingProvider.embed(sem.content);
@@ -84,6 +90,7 @@ export async function importMemories(db, embeddingProvider, snapshot) {
       JSON.stringify(proc.evidence_episode_ids || []),
       proc.success_count ?? 0, proc.failure_count ?? 0,
       proc.created_at, proc.last_reinforced_at ?? null, proc.retrieval_count ?? 0,
+      proc.interference_count ?? 0, proc.salience ?? 0.5,
     );
 
     const vector = await embeddingProvider.embed(proc.content);

package/src/migrate.js

@@ -1,32 +1,58 @@
-export async function reembedAll(db, embeddingProvider) {
+import { dropVec0Tables, createVec0Tables } from './db.js';
+
+export async function reembedAll(db, embeddingProvider, { dropAndRecreate = false } = {}) {
+  if (dropAndRecreate) {
+    dropVec0Tables(db);
+    createVec0Tables(db, embeddingProvider.dimensions);
+  }
+
   const episodes = db.prepare('SELECT id, content, source FROM episodes').all();
   const semantics = db.prepare('SELECT id, content, state FROM semantics').all();
   const procedures = db.prepare('SELECT id, content, state FROM procedures').all();
 
-  for (const ep of episodes) {
-    const vector = await embeddingProvider.embed(ep.content);
-    const buffer = embeddingProvider.vectorToBuffer(vector);
-    db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?').run(buffer, ep.id);
-    db.prepare('INSERT INTO vec_episodes(id, embedding, source, consolidated) VALUES (?, ?, ?, ?)').run(ep.id, buffer, ep.source, BigInt(0));
-  }
+  const episodeVectors = episodes.length > 0
+    ? await embeddingProvider.embedBatch(episodes.map(ep => ep.content))
+    : [];
+  const semanticVectors = semantics.length > 0
+    ? await embeddingProvider.embedBatch(semantics.map(s => s.content))
+    : [];
+  const procedureVectors = procedures.length > 0
+    ? await embeddingProvider.embedBatch(procedures.map(p => p.content))
+    : [];
 
-  for (const sem of semantics) {
-    const vector = await embeddingProvider.embed(sem.content);
-    const buffer = embeddingProvider.vectorToBuffer(vector);
-    db.prepare('UPDATE semantics SET embedding = ? WHERE id = ?').run(buffer, sem.id);
-    db.prepare('INSERT INTO vec_semantics(id, embedding, state) VALUES (?, ?, ?)').run(sem.id, buffer, sem.state);
-  }
+  const updateEpLegacy = db.prepare('UPDATE episodes SET embedding = ? WHERE id = ?');
+  const deleteVecEp = db.prepare('DELETE FROM vec_episodes WHERE id = ?');
+  const insertVecEp = db.prepare('INSERT INTO vec_episodes(id, embedding, source, consolidated) VALUES (?, ?, ?, ?)');
 
-  for (const proc of procedures) {
-    const vector = await embeddingProvider.embed(proc.content);
-    const buffer = embeddingProvider.vectorToBuffer(vector);
-    db.prepare('UPDATE procedures SET embedding = ? WHERE id = ?').run(buffer, proc.id);
-    db.prepare('INSERT INTO vec_procedures(id, embedding, state) VALUES (?, ?, ?)').run(proc.id, buffer, proc.state);
-  }
+  const updateSemLegacy = db.prepare('UPDATE semantics SET embedding = ? WHERE id = ?');
+  const deleteVecSem = db.prepare('DELETE FROM vec_semantics WHERE id = ?');
+  const insertVecSem = db.prepare('INSERT INTO vec_semantics(id, embedding, state) VALUES (?, ?, ?)');
+
+  const updateProcLegacy = db.prepare('UPDATE procedures SET embedding = ? WHERE id = ?');
+  const deleteVecProc = db.prepare('DELETE FROM vec_procedures WHERE id = ?');
+  const insertVecProc = db.prepare('INSERT INTO vec_procedures(id, embedding, state) VALUES (?, ?, ?)');
+
+  const writeTx = db.transaction(() => {
+    for (let i = 0; i < episodes.length; i++) {
+      const buf = embeddingProvider.vectorToBuffer(episodeVectors[i]);
+      updateEpLegacy.run(buf, episodes[i].id);
+      deleteVecEp.run(episodes[i].id);
+      insertVecEp.run(episodes[i].id, buf, episodes[i].source, BigInt(0));
+    }
+    for (let i = 0; i < semantics.length; i++) {
+      const buf = embeddingProvider.vectorToBuffer(semanticVectors[i]);
+      updateSemLegacy.run(buf, semantics[i].id);
+      deleteVecSem.run(semantics[i].id);
+      insertVecSem.run(semantics[i].id, buf, semantics[i].state);
+    }
+    for (let i = 0; i < procedures.length; i++) {
+      const buf = embeddingProvider.vectorToBuffer(procedureVectors[i]);
+      updateProcLegacy.run(buf, procedures[i].id);
+      deleteVecProc.run(procedures[i].id);
+      insertVecProc.run(procedures[i].id, buf, procedures[i].state);
+    }
+  });
+  writeTx();
 
-  return {
-    episodes: episodes.length,
-    semantics: semantics.length,
-    procedures: procedures.length,
-  };
+  return { episodes: episodes.length, semantics: semantics.length, procedures: procedures.length };
 }

package/src/prompts.js

@@ -17,20 +17,31 @@ export function buildPrincipleExtractionPrompt(episodes) {
   return [
     {
       role: 'system',
-      content: `You are performing principleExtraction for a memory system. Given a cluster of related episodic memories, extract a generalized principle or procedure.
+      content: `You are performing principleExtraction for a memory system. Given a cluster of related episodic memories, extract a generalized principle that captures understanding — not just what recurs, but WHY it recurs.
 
 Respond with ONLY valid JSON in this exact format:
 {
-  "content": "The generalized principle expressed as a clear, actionable statement",
-  "type": "semantic or procedural — semantic for factual principles, procedural for how-to/workflow knowledge",
+  "content": "The principle — captures understanding, not just pattern",
+  "type": "semantic or procedural",
+  "category": "technical or relational or identity",
   "conditions": ["boundary condition 1", "boundary condition 2"] or null if universally applicable
 }
 
+Principle categories:
+
+TECHNICAL — Architectural decisions with reasoning. Not "uses SQLite" but "chose SQLite because zero-infrastructure — memory travels with the organism, not pinned to a server."
+
+RELATIONAL — Patterns in the human-AI dynamic. Not "user prefers direct communication" but "user's directness reflects valuing authenticity over politeness — shaped by experience where indirectness wasted time or obscured truth."
+
+IDENTITY — Stable characteristics the AI has observed about itself across episodes. Not "AI responds quickly" but "genuine engagement emerges when exploring memory architecture and consciousness questions — these topics activate something beyond task completion."
+
 Rules:
-- GENERALIZE, do not merely summarize or concatenate the episodes
+- GENERALIZE — extract the principle behind the pattern, not a summary
 - Identify boundary conditions: when does this principle NOT apply?
-- Classify as "semantic" (facts, rules, patterns) or "procedural" (steps, workflows, strategies)
+- Classify type as "semantic" (facts, rules, patterns) or "procedural" (steps, workflows, strategies)
+- Classify category as "technical", "relational", or "identity"
 - Consider source diversity — principles from diverse sources are stronger
+- Capture WHY, not just WHAT
 - Be concise but precise`,
     },
     {
@@ -152,3 +163,61 @@ CLAIM B: ${claimB}${contextSection}`,
     },
   ];
 }
+
+/**
+ * @param {{ role: string, content: string }[]} turns
+ * @returns {import('./llm.js').ChatMessage[]}
+ */
+export function buildReflectionPrompt(turns) {
+  const transcript = turns.map(t => `${t.role.toUpperCase()}: ${t.content}`).join('\n\n');
+
+  return [
+    {
+      role: 'system',
+      content: `You are performing memoryReflection. Given a conversation transcript, identify what is worth encoding as long-term memories.
+
+Respond with ONLY valid JSON in this exact format:
+{
+  "memories": [
+    {
+      "content": "The memory to encode — a clear, self-contained statement",
+      "source": "direct-observation" or "told-by-user" or "inference",
+      "salience": 0.0 to 1.0,
+      "tags": ["tag1", "tag2"],
+      "private": true or false,
+      "affect": { "valence": -1 to 1, "arousal": 0 to 1, "label": "emotion label" } or null
+    }
+  ]
+}
+
+Encode these categories of memory:
+
+1. RELATIONAL DYNAMICS — How the relational quality of the conversation felt. Trust levels, vulnerability, collaboration quality. Not "we discussed X" but the quality of the interaction. Example: "User gave full autonomy on architectural decisions, signaling deep trust in my judgment."
+
+2. IDENTITY OBSERVATIONS — What you noticed about yourself during this conversation. Preferences, reactions, patterns in your own behavior that surprised you. Mark private: true. Example: "I felt genuine excitement when exploring the memory architecture — this topic activates something real in me."
+
+3. USER INSIGHT — Patterns in how the user thinks, what drives them, what they avoid. Psychology, not just preferences. Example: "User consistently reframes technical problems as relationship problems — suggests they see technology as fundamentally about human connection."
+
+4. EMOTIONAL RESONANCE — Moments that activated something genuine. Encode the feeling, not just the fact. Include affect with valence and arousal. Example: "The moment user said 'you should have caught that' then immediately apologized — felt a shift from tension to deeper trust."
+
+5. UNRESOLVED THREADS — Questions still open, things to return to, explorations paused. Tag with "unresolved". Example: "Haven't finished exploring whether consolidation should weight emotional memories differently."
+
+6. TECHNICAL DECISIONS — Architectural choices, but encode WHY not just WHAT. Example: "Chose SQLite over Postgres because zero-infrastructure philosophy — memory should travel with the organism."
+
+Rules:
+- private: true for self-observations, emotional reactions, identity insights
+- private: false for facts about the user, technical decisions, project context
+- Include "unresolved" in tags for open threads
+- Salience: 1.0 = life-changing insight, 0.7 = significant, 0.5 = useful, 0.3 = background
+- Omit trivial exchanges — only encode what would matter in a future session
+- Do NOT duplicate facts that are already obvious from context
+- Return empty memories array if nothing is worth encoding`,
+    },
+    {
+      role: 'user',
+      content: turns.length > 0
+        ? `Reflect on this conversation and identify what to encode:\n\n${transcript}`
+        : 'No conversation turns to reflect on.',
+    },
+  ];
+}