@mneme-ai/core 0.23.0 → 0.25.0

package/dist/store/schema.d.ts

@@ -5,7 +5,8 @@
  * Phase 2 — entities, entity_clusters
  * Phase 3 — incidents, correlations
  * Phase 4 — graph_snapshots (for temporal viz)
+ * Phase 5 — htc (Hierarchical Token Cache) — abstracts, clusters, memoir
  */
-export declare const SCHEMA_VERSION = 3;
-export declare const SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS meta (\n  key TEXT PRIMARY KEY,\n  value TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS commits (\n  hash TEXT PRIMARY KEY,\n  short_hash TEXT NOT NULL,\n  author_name TEXT NOT NULL,\n  author_email TEXT NOT NULL,\n  author_date TEXT NOT NULL,\n  committer_date TEXT NOT NULL,\n  subject TEXT NOT NULL,\n  body TEXT NOT NULL,\n  parents TEXT NOT NULL,\n  pr_number INTEGER,\n  pr_title TEXT,\n  pr_body TEXT,\n  issue_refs TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_commits_author_date ON commits(author_date);\nCREATE INDEX IF NOT EXISTS idx_commits_pr ON commits(pr_number);\n\nCREATE TABLE IF NOT EXISTS file_changes (\n  commit_hash TEXT NOT NULL,\n  path TEXT NOT NULL,\n  change_kind TEXT NOT NULL,\n  insertions INTEGER NOT NULL DEFAULT 0,\n  deletions INTEGER NOT NULL DEFAULT 0,\n  PRIMARY KEY (commit_hash, path),\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\nCREATE INDEX IF NOT EXISTS idx_file_changes_path ON file_changes(path);\n\nCREATE TABLE IF NOT EXISTS chunks (\n  id TEXT PRIMARY KEY,\n  commit_hash TEXT NOT NULL,\n  kind TEXT NOT NULL,\n  text TEXT NOT NULL,\n  embedding BLOB,\n  embedding_model TEXT,\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\nCREATE INDEX IF NOT EXISTS idx_chunks_commit ON chunks(commit_hash);\nCREATE INDEX IF NOT EXISTS idx_chunks_kind ON chunks(kind);\n\n-- FTS5 with trigram tokenizer \u2014 works for Thai, CJK, Arabic, and any\n-- non-space-separated language. The tradeoff vs porter is that BM25 is\n-- slightly less precise for English, but the gain in cross-language\n-- coverage is enormous. Also: trigram makes substring search free.\nCREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(\n  id UNINDEXED,\n  commit_hash UNINDEXED,\n  text,\n  tokenize = 'trigram'\n);\n\n-- Phase 2: entity graph\nCREATE TABLE IF NOT EXISTS entities (\n  id TEXT PRIMARY KEY,\n  kind TEXT NOT NULL,\n  name TEXT NOT NULL,\n  file_path TEXT NOT NULL,\n  start_line INTEGER NOT NULL,\n  end_line INTEGER NOT NULL,\n  signature TEXT,\n  language TEXT NOT NULL,\n  embedding BLOB\n);\nCREATE INDEX IF NOT EXISTS idx_entities_file ON entities(file_path);\nCREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);\n\n-- Phase 3: incidents and correlations (the moat)\nCREATE TABLE IF NOT EXISTS incidents (\n  id TEXT PRIMARY KEY,\n  source TEXT NOT NULL,\n  external_id TEXT,\n  title TEXT NOT NULL,\n  occurred_at TEXT NOT NULL,\n  resolved_at TEXT,\n  severity TEXT NOT NULL,\n  affected_files TEXT,\n  stack_frames TEXT,\n  url TEXT,\n  metadata TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_incidents_occurred ON incidents(occurred_at);\nCREATE INDEX IF NOT EXISTS idx_incidents_source ON incidents(source);\n\nCREATE TABLE IF NOT EXISTS correlations (\n  id TEXT PRIMARY KEY,\n  from_kind TEXT NOT NULL,\n  from_id TEXT NOT NULL,\n  to_kind TEXT NOT NULL,\n  to_id TEXT NOT NULL,\n  weight REAL NOT NULL,\n  reason TEXT NOT NULL,\n  evidence TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_corr_from ON correlations(from_kind, from_id);\nCREATE INDEX IF NOT EXISTS idx_corr_to ON correlations(to_kind, to_id);\n\n-- Phase 4: temporal graph snapshots\nCREATE TABLE IF NOT EXISTS graph_snapshots (\n  id TEXT PRIMARY KEY,\n  taken_at TEXT NOT NULL,\n  payload BLOB NOT NULL\n);\n\n-- WILD #1: AI-synthesized notes for commits with poor messages.\n-- The original commit is never modified. The synthesized note is searched\n-- alongside the original chunks but always marked as kind='synthesized'\n-- so users can verify against the underlying diff.\nCREATE TABLE IF NOT EXISTS synthesized_notes (\n  commit_hash TEXT PRIMARY KEY,\n  note TEXT NOT NULL,\n  model TEXT NOT NULL,\n  diff_chars INTEGER NOT NULL,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\n\n-- Phase 4 - Wisdom Mutant Engine\n-- The \"always-learning\" loop. Three append-only tables that record what the\n-- system saw, what it tried, and how well it worked. Per-repo, never leaves\n-- the machine. Honest contract: every recommendation in mneme adapt and\n-- every threshold in mneme ask is traceable back to a row here.\n\n-- Every query records its result-set fingerprint. was_helpful is set later,\n-- either by the user (--feedback up/down) or by an implicit signal (the\n-- query was not re-run with a refined version within 60s, or one of the\n-- returned commits was navigated to via mneme why).\nCREATE TABLE IF NOT EXISTS wisdom_feedback (\n  id TEXT PRIMARY KEY,\n  query TEXT NOT NULL,\n  result_hashes TEXT NOT NULL,        -- JSON array of commit hashes returned\n  top_score REAL,                     -- top RRF score at query time\n  was_helpful INTEGER,                -- 1=yes, 0=no, NULL=unknown\n  source TEXT NOT NULL,               -- 'explicit' | 'implicit-reuse' | 'implicit-revisit'\n  semantic_weight REAL,               -- the search config that produced this result\n  min_sem_cosine REAL,\n  rrf_k INTEGER,\n  created_at TEXT NOT NULL\n);\nCREATE INDEX IF NOT EXISTS idx_feedback_created ON wisdom_feedback(created_at);\nCREATE INDEX IF NOT EXISTS idx_feedback_helpful ON wisdom_feedback(was_helpful);\n\n-- The current calibration of search/retrieval knobs. One row per knob.\n-- Updated by the auto-calibrator after evaluating against the feedback set.\nCREATE TABLE IF NOT EXISTS wisdom_calibration (\n  key TEXT PRIMARY KEY,\n  value TEXT NOT NULL,                -- JSON\n  metric_value REAL,                  -- the metric this calibration achieved\n  metric_name TEXT,                   -- e.g. \"hit_rate\", \"recall_at_3\"\n  calibrated_at TEXT NOT NULL,\n  notes TEXT\n);\n\n-- Append-only history of self-eval runs. Lets the user (and any auditor)\n-- see whether quality is trending up or down over time.\nCREATE TABLE IF NOT EXISTS wisdom_eval_run (\n  id INTEGER PRIMARY KEY AUTOINCREMENT,\n  ran_at TEXT NOT NULL,\n  variant TEXT NOT NULL,              -- 'baseline' | 'reranked' | 'calibrated' | ...\n  recall_at_3 REAL NOT NULL,\n  mrr REAL NOT NULL,\n  ndcg_at_10 REAL,\n  hit_rate REAL NOT NULL,\n  semantic_weight REAL,\n  min_sem_cosine REAL,\n  rrf_k INTEGER,\n  num_queries INTEGER NOT NULL,\n  notes TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_eval_ran_at ON wisdom_eval_run(ran_at);\n";
+export declare const SCHEMA_VERSION = 4;
+export declare const SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS meta (\n  key TEXT PRIMARY KEY,\n  value TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS commits (\n  hash TEXT PRIMARY KEY,\n  short_hash TEXT NOT NULL,\n  author_name TEXT NOT NULL,\n  author_email TEXT NOT NULL,\n  author_date TEXT NOT NULL,\n  committer_date TEXT NOT NULL,\n  subject TEXT NOT NULL,\n  body TEXT NOT NULL,\n  parents TEXT NOT NULL,\n  pr_number INTEGER,\n  pr_title TEXT,\n  pr_body TEXT,\n  issue_refs TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_commits_author_date ON commits(author_date);\nCREATE INDEX IF NOT EXISTS idx_commits_pr ON commits(pr_number);\n\nCREATE TABLE IF NOT EXISTS file_changes (\n  commit_hash TEXT NOT NULL,\n  path TEXT NOT NULL,\n  change_kind TEXT NOT NULL,\n  insertions INTEGER NOT NULL DEFAULT 0,\n  deletions INTEGER NOT NULL DEFAULT 0,\n  PRIMARY KEY (commit_hash, path),\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\nCREATE INDEX IF NOT EXISTS idx_file_changes_path ON file_changes(path);\n\nCREATE TABLE IF NOT EXISTS chunks (\n  id TEXT PRIMARY KEY,\n  commit_hash TEXT NOT NULL,\n  kind TEXT NOT NULL,\n  text TEXT NOT NULL,\n  embedding BLOB,\n  embedding_model TEXT,\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\nCREATE INDEX IF NOT EXISTS idx_chunks_commit ON chunks(commit_hash);\nCREATE INDEX IF NOT EXISTS idx_chunks_kind ON chunks(kind);\n\n-- FTS5 with trigram tokenizer \u2014 works for Thai, CJK, Arabic, and any\n-- non-space-separated language. The tradeoff vs porter is that BM25 is\n-- slightly less precise for English, but the gain in cross-language\n-- coverage is enormous. Also: trigram makes substring search free.\nCREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(\n  id UNINDEXED,\n  commit_hash UNINDEXED,\n  text,\n  tokenize = 'trigram'\n);\n\n-- Phase 2: entity graph\nCREATE TABLE IF NOT EXISTS entities (\n  id TEXT PRIMARY KEY,\n  kind TEXT NOT NULL,\n  name TEXT NOT NULL,\n  file_path TEXT NOT NULL,\n  start_line INTEGER NOT NULL,\n  end_line INTEGER NOT NULL,\n  signature TEXT,\n  language TEXT NOT NULL,\n  embedding BLOB\n);\nCREATE INDEX IF NOT EXISTS idx_entities_file ON entities(file_path);\nCREATE INDEX IF NOT EXISTS idx_entities_name ON entities(name);\n\n-- Phase 3: incidents and correlations (the moat)\nCREATE TABLE IF NOT EXISTS incidents (\n  id TEXT PRIMARY KEY,\n  source TEXT NOT NULL,\n  external_id TEXT,\n  title TEXT NOT NULL,\n  occurred_at TEXT NOT NULL,\n  resolved_at TEXT,\n  severity TEXT NOT NULL,\n  affected_files TEXT,\n  stack_frames TEXT,\n  url TEXT,\n  metadata TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_incidents_occurred ON incidents(occurred_at);\nCREATE INDEX IF NOT EXISTS idx_incidents_source ON incidents(source);\n\nCREATE TABLE IF NOT EXISTS correlations (\n  id TEXT PRIMARY KEY,\n  from_kind TEXT NOT NULL,\n  from_id TEXT NOT NULL,\n  to_kind TEXT NOT NULL,\n  to_id TEXT NOT NULL,\n  weight REAL NOT NULL,\n  reason TEXT NOT NULL,\n  evidence TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_corr_from ON correlations(from_kind, from_id);\nCREATE INDEX IF NOT EXISTS idx_corr_to ON correlations(to_kind, to_id);\n\n-- Phase 4: temporal graph snapshots\nCREATE TABLE IF NOT EXISTS graph_snapshots (\n  id TEXT PRIMARY KEY,\n  taken_at TEXT NOT NULL,\n  payload BLOB NOT NULL\n);\n\n-- WILD #1: AI-synthesized notes for commits with poor messages.\n-- The original commit is never modified. The synthesized note is searched\n-- alongside the original chunks but always marked as kind='synthesized'\n-- so users can verify against the underlying diff.\nCREATE TABLE IF NOT EXISTS synthesized_notes (\n  commit_hash TEXT PRIMARY KEY,\n  note TEXT NOT NULL,\n  model TEXT NOT NULL,\n  diff_chars INTEGER NOT NULL,\n  created_at TEXT NOT NULL,\n  FOREIGN KEY (commit_hash) REFERENCES commits(hash) ON DELETE CASCADE\n);\n\n-- Phase 4 - Wisdom Mutant Engine\n-- The \"always-learning\" loop. Three append-only tables that record what the\n-- system saw, what it tried, and how well it worked. Per-repo, never leaves\n-- the machine. Honest contract: every recommendation in mneme adapt and\n-- every threshold in mneme ask is traceable back to a row here.\n\n-- Every query records its result-set fingerprint. was_helpful is set later,\n-- either by the user (--feedback up/down) or by an implicit signal (the\n-- query was not re-run with a refined version within 60s, or one of the\n-- returned commits was navigated to via mneme why).\nCREATE TABLE IF NOT EXISTS wisdom_feedback (\n  id TEXT PRIMARY KEY,\n  query TEXT NOT NULL,\n  result_hashes TEXT NOT NULL,        -- JSON array of commit hashes returned\n  top_score REAL,                     -- top RRF score at query time\n  was_helpful INTEGER,                -- 1=yes, 0=no, NULL=unknown\n  source TEXT NOT NULL,               -- 'explicit' | 'implicit-reuse' | 'implicit-revisit'\n  semantic_weight REAL,               -- the search config that produced this result\n  min_sem_cosine REAL,\n  rrf_k INTEGER,\n  created_at TEXT NOT NULL\n);\nCREATE INDEX IF NOT EXISTS idx_feedback_created ON wisdom_feedback(created_at);\nCREATE INDEX IF NOT EXISTS idx_feedback_helpful ON wisdom_feedback(was_helpful);\n\n-- The current calibration of search/retrieval knobs. One row per knob.\n-- Updated by the auto-calibrator after evaluating against the feedback set.\nCREATE TABLE IF NOT EXISTS wisdom_calibration (\n  key TEXT PRIMARY KEY,\n  value TEXT NOT NULL,                -- JSON\n  metric_value REAL,                  -- the metric this calibration achieved\n  metric_name TEXT,                   -- e.g. \"hit_rate\", \"recall_at_3\"\n  calibrated_at TEXT NOT NULL,\n  notes TEXT\n);\n\n-- Append-only history of self-eval runs. Lets the user (and any auditor)\n-- see whether quality is trending up or down over time.\nCREATE TABLE IF NOT EXISTS wisdom_eval_run (\n  id INTEGER PRIMARY KEY AUTOINCREMENT,\n  ran_at TEXT NOT NULL,\n  variant TEXT NOT NULL,              -- 'baseline' | 'reranked' | 'calibrated' | ...\n  recall_at_3 REAL NOT NULL,\n  mrr REAL NOT NULL,\n  ndcg_at_10 REAL,\n  hit_rate REAL NOT NULL,\n  semantic_weight REAL,\n  min_sem_cosine REAL,\n  rrf_k INTEGER,\n  num_queries INTEGER NOT NULL,\n  notes TEXT\n);\nCREATE INDEX IF NOT EXISTS idx_eval_ran_at ON wisdom_eval_run(ran_at);\n\n-- Phase 5 \u2014 Hierarchical Token Cache (HTC)\n-- The world-first compression-as-storage layer. Mneme pre-compresses an\n-- entire codebase's git history into LLM-consumable form at index-time.\n-- 50,000 commits fit in one LLM prompt; token cost paid ONCE; reused forever.\n--\n-- Three layers:\n--   Layer 1 \u2014 htc_abstracts: ~30 tok/commit, LLM-generated once, cached\n--   Layer 2 \u2014 htc_clusters:  ~100 tok/cluster, summarized from Layer 1\n--   Layer 3 \u2014 htc_memoir:    ~500 tok, generated from Layer 2\n\nCREATE TABLE IF NOT EXISTS htc_abstracts (\n  commit_hash TEXT PRIMARY KEY REFERENCES commits(hash) ON DELETE CASCADE,\n  abstract TEXT NOT NULL,\n  token_count INTEGER NOT NULL,\n  generated_at TEXT NOT NULL,\n  generator TEXT NOT NULL,\n  generation_ms INTEGER NOT NULL\n);\nCREATE INDEX IF NOT EXISTS idx_htc_abstracts_generated ON htc_abstracts(generated_at);\n\nCREATE TABLE IF NOT EXISTS htc_clusters (\n  cluster_id TEXT PRIMARY KEY,\n  label TEXT NOT NULL,\n  summary TEXT NOT NULL,\n  member_hashes TEXT NOT NULL,\n  token_count INTEGER NOT NULL,\n  generated_at TEXT NOT NULL,\n  generator TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS htc_memoir (\n  id INTEGER PRIMARY KEY,\n  narrative TEXT NOT NULL,\n  total_commits INTEGER NOT NULL,\n  total_clusters INTEGER NOT NULL,\n  token_count INTEGER NOT NULL,\n  generated_at TEXT NOT NULL,\n  generator TEXT NOT NULL\n);\n";
 //# sourceMappingURL=schema.d.ts.map

package/dist/store/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/store/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,eAAO,MAAM,UAAU,upMAgLtB,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/store/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,IAAI,CAAC;AAEhC,eAAO,MAAM,UAAU,4hPAwNtB,CAAC"}

package/dist/store/schema.js

@@ -5,8 +5,9 @@
  * Phase 2 — entities, entity_clusters
  * Phase 3 — incidents, correlations
  * Phase 4 — graph_snapshots (for temporal viz)
+ * Phase 5 — htc (Hierarchical Token Cache) — abstracts, clusters, memoir
  */
-export const SCHEMA_VERSION = 3;
+export const SCHEMA_VERSION = 4;
 export const SCHEMA_SQL = `
 CREATE TABLE IF NOT EXISTS meta (
   key TEXT PRIMARY KEY,
@@ -183,5 +184,45 @@ CREATE TABLE IF NOT EXISTS wisdom_eval_run (
   notes TEXT
 );
 CREATE INDEX IF NOT EXISTS idx_eval_ran_at ON wisdom_eval_run(ran_at);
+
+-- Phase 5 — Hierarchical Token Cache (HTC)
+-- The world-first compression-as-storage layer. Mneme pre-compresses an
+-- entire codebase's git history into LLM-consumable form at index-time.
+-- 50,000 commits fit in one LLM prompt; token cost paid ONCE; reused forever.
+--
+-- Three layers:
+--   Layer 1 — htc_abstracts: ~30 tok/commit, LLM-generated once, cached
+--   Layer 2 — htc_clusters:  ~100 tok/cluster, summarized from Layer 1
+--   Layer 3 — htc_memoir:    ~500 tok, generated from Layer 2
+
+CREATE TABLE IF NOT EXISTS htc_abstracts (
+  commit_hash TEXT PRIMARY KEY REFERENCES commits(hash) ON DELETE CASCADE,
+  abstract TEXT NOT NULL,
+  token_count INTEGER NOT NULL,
+  generated_at TEXT NOT NULL,
+  generator TEXT NOT NULL,
+  generation_ms INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_htc_abstracts_generated ON htc_abstracts(generated_at);
+
+CREATE TABLE IF NOT EXISTS htc_clusters (
+  cluster_id TEXT PRIMARY KEY,
+  label TEXT NOT NULL,
+  summary TEXT NOT NULL,
+  member_hashes TEXT NOT NULL,
+  token_count INTEGER NOT NULL,
+  generated_at TEXT NOT NULL,
+  generator TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS htc_memoir (
+  id INTEGER PRIMARY KEY,
+  narrative TEXT NOT NULL,
+  total_commits INTEGER NOT NULL,
+  total_clusters INTEGER NOT NULL,
+  token_count INTEGER NOT NULL,
+  generated_at TEXT NOT NULL,
+  generator TEXT NOT NULL
+);
 `;
 //# sourceMappingURL=schema.js.map

package/dist/store/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/store/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC,MAAM,CAAC,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgLzB,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/store/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC,MAAM,CAAC,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwNzB,CAAC"}

package/dist/store/sqlite.test.js

@@ -32,7 +32,7 @@ const sampleCommit = (over = {}) => ({
 });
 describe("MnemeStore — schema + roundtrip", () => {
     it("initializes with schema_version meta key", () => {
-        expect(store.getMeta("schema_version")).toBe("3");
+        expect(store.getMeta("schema_version")).toBe("4");
     });
     it("upserts and retrieves a commit losslessly", () => {
         const c = sampleCommit();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mneme-ai/core",
-  "version": "0.23.0",
+  "version": "0.25.0",
   "description": "Core indexing, retrieval, and graph engine for Mneme",
   "type": "module",
   "main": "./dist/index.js",