@jungjaehoon/mama-core 1.4.2 → 1.6.0

package/README.md

@@ -1,6 +1,6 @@
 # @jungjaehoon/mama-core
 
-Shared core modules for MAMA (Memory-Augmented MCP Assistant).
+Shared memory, provenance, graph, and context substrate for MAMA.
 
 ## What is MAMA Core?
 
@@ -10,7 +10,9 @@ MAMA Core is a shared package containing the fundamental modules used by all MAM
 - **claude-code-plugin**: Claude Code plugin
 - **standalone**: Standalone HTTP server
 
-This package provides embedding generation, database management, decision tracking, and other core functionality without the transport layer (MCP/HTTP).
+This package provides embedding generation, database management, scoped memory, provenance, raw
+evidence refs, graph/entity helpers, and worker context primitives without the transport layer
+(MCP/HTTP).
 
 ## Installation
 
@@ -78,7 +80,7 @@ const mamaApi = require('@jungjaehoon/mama-core/mama-api');
 
 - **memory/api** - Scoped memory operations
   - `saveMemory(input)` - Save typed memory with scopes and optional event_date (preference, fact, decision, lesson, constraint)
-  - `recallMemory(query, options)` - Truth-aware recall with scope filtering
+  - `recallMemory(query, options)` - Truth-aware recall with scope filtering, strictness modes, and retrieval diagnostics
   - `buildProfile(scopes)` - Build memory profile (static/dynamic/evidence)
   - `ingestMemory(input)` - Ingest raw content as memory
   - `ingestConversation(input)` - Decompose conversations into typed memory units via optional LLM extraction
@@ -102,14 +104,39 @@ const mamaApi = require('@jungjaehoon/mama-core/mama-api');
 - **memory/channel-summary-state-store** - Channel state management
   - `recordChannelAudit(input)` - Accumulate audit outcomes into channel state
 
+- **memory/provenance** - Trusted memory provenance
+  - Preserves compact runtime origin metadata in `provenance_json`
+  - Keeps underlying raw/memory/case/entity refs in `source_refs_json`
+  - Ignores caller-supplied untrusted provenance on public paths
+
+- **model-runs** - Agent execution lineage
+  - `model-runs/store` records model run lifecycle and replay metadata
+  - `model-runs/tool-trace-store` records tool calls for later audit and reconstruction
+
+- **connectors/raw-query** - Unified raw evidence reads
+  - Query raw connector rows through scope-aware filters
+  - Preserve raw source ids for downstream provenance
+
+- **agent-situation** - Worker situation packets
+  - Build, rank, cache, insert, and read append-only situation packets for worker context
+
+- **agent-graph** - Worker graph/entity helpers
+  - Query graph neighborhoods, resolve entities, and write aliases with visibility checks
+
 ### Core API (Legacy)
 
 - **mama-api** - High-level API interface (wraps memory API)
   - `save(decision)` - Save decision
   - `recall(topic)` - Retrieve decision history
-  - `suggest(query)` - Semantic search with hybrid FTS5 + vector + recency
+  - `suggest(query, options)` - Semantic search with hybrid FTS5 + vector + recency, strictness controls, and diagnostics
   - `updateOutcome(id, outcome)` - Update decision outcome
 
+- **search/search-quality** - Search option normalization
+  - `normalizeSearchQualityOptions(options)` - Normalize recall/balanced/strict thresholds and confirmation requirements
+
+- **edges** - Twin edge ledger
+  - Validate refs and store durable edges between memory, raw, entity, case, and packet refs
+
 - **decision-tracker** - Decision graph management
   - `learnDecision(decision)` - Learn from decision
   - `createEdgesFromReasoning(reasoning)` - Parse decision links
@@ -135,7 +162,7 @@ const mamaApi = require('@jungjaehoon/mama-core/mama-api');
 ## Dependencies
 
 - **@huggingface/transformers** - Local embedding generation
-- **node:sqlite** - Built-in SQLite runtime (Node.js 22+)
+- **better-sqlite3** - SQLite runtime with FTS5 support
 - **Pure-TS cosine similarity** - Vector search (no native extensions)
 
 ## Development
@@ -153,7 +180,7 @@ pnpm test:watch
 
 ## Test Coverage
 
-- 72 unit tests across 17 test files
+- 120+ unit/integration test files in `packages/mama-core/tests`
 - 100% passing
 - Tests cover:
   - Config loader, database initialization, module exports
@@ -161,6 +188,9 @@ pnpm test:watch
   - Truth store, evolution engine, scope schema
   - Channel summary, channel summary state
   - Event store, finding store, bootstrap builder
+  - Memory provenance, model runs, tool traces
+  - Twin edges, raw query, agent situation, agent graph
+  - Strict search diagnostics and rollup provenance
   - Legacy shim compatibility
 
 ## Architecture
@@ -175,7 +205,7 @@ packages/mama-core/
 │   ├── db-manager.ts         # Database management
 │   ├── mama-api.ts           # High-level API (wraps memory API)
 │   ├── db-adapter/           # Database adapter (SQLite)
-│   └── memory/               # Memory infrastructure
+│   ├── memory/               # Memory infrastructure
 │       ├── types.ts          # MemoryRecord, MemoryScopeRef, RecallBundle, etc.
 │       ├── api.ts            # saveMemory, recallMemory, buildProfile, etc.
 │       ├── truth-store.ts    # Truth projection layer
@@ -188,13 +218,19 @@ packages/mama-core/
 │       ├── channel-summary-state-store.ts # Channel state reducer
 │       ├── bootstrap-builder.ts           # Memory agent bootstrap
 │       └── profile-builder.ts             # Profile classification
-├── db/migrations/            # SQLite migrations (001-023)
-└── tests/                    # 16 test files, 59 tests
+│   ├── model-runs/           # Model run + tool trace lineage
+│   ├── connectors/           # Raw connector query/index helpers
+│   ├── edges/                # Twin edge ledger
+│   ├── agent-situation/      # Worker situation packets
+│   ├── agent-graph/          # Worker graph/entity helpers
+│   └── search/               # Search quality option normalization
+├── db/migrations/            # SQLite migrations (001-036)
+└── tests/                    # Unit and integration tests
 ```
 
 ## Migration Files
 
-Database migrations are included in `db/migrations/` (001-023):
+Database migrations are included in `db/migrations/` (001-036):
 
 - 001-013: Core schema (decisions, embeddings, graph edges)
 - 014: Add is_static column
@@ -204,6 +240,8 @@ Database migrations are included in `db/migrations/` (001-023):
 - 019-020: Memory events and audit findings
 - 021: Memory truth projection
 - 022-023: Channel summaries and state
+- 024-036: event dates, connector memory kinds, canonical entities, case-first memory, wiki/index
+  provenance, model/tool traces, twin edges, and agent situation packets
 
 ## License
 

package/db/migrations/026-create-canonical-entity-tables.sql

@@ -0,0 +1,99 @@
+CREATE TABLE IF NOT EXISTS entity_nodes (
+  id TEXT PRIMARY KEY,
+  kind TEXT NOT NULL CHECK (kind IN ('project', 'person', 'organization', 'work_item')),
+  preferred_label TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'merged', 'archived')),
+  scope_kind TEXT CHECK (scope_kind IN ('project', 'channel', 'user', 'global')),
+  scope_id TEXT,
+  merged_into TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (merged_into) REFERENCES entity_nodes(id),
+  CHECK ((scope_kind = 'global' AND scope_id IS NULL) OR (scope_kind != 'global'))
+);
+
+CREATE TABLE IF NOT EXISTS entity_aliases (
+  id TEXT PRIMARY KEY,
+  entity_id TEXT NOT NULL,
+  label TEXT NOT NULL,
+  normalized_label TEXT NOT NULL,
+  lang TEXT,
+  script TEXT,
+  label_type TEXT NOT NULL CHECK (label_type IN ('pref', 'alt', 'hidden', 'source_native')),
+  source_type TEXT NOT NULL,
+  source_ref TEXT,
+  confidence REAL,
+  status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'suppressed')),
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (entity_id) REFERENCES entity_nodes(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS entity_observations (
+  id TEXT PRIMARY KEY,
+  entity_kind_hint TEXT CHECK (entity_kind_hint IN ('project', 'person', 'organization', 'work_item')),
+  surface_form TEXT NOT NULL,
+  normalized_form TEXT NOT NULL,
+  lang TEXT,
+  script TEXT,
+  context_summary TEXT,
+  related_surface_forms TEXT NOT NULL DEFAULT '[]',
+  timestamp_observed INTEGER,
+  scope_kind TEXT CHECK (scope_kind IN ('project', 'channel', 'user', 'global')),
+  scope_id TEXT,
+  extractor_version TEXT NOT NULL,
+  embedding_model_version TEXT,
+  source_connector TEXT NOT NULL,
+  source_raw_db_ref TEXT,
+  source_raw_record_id TEXT NOT NULL,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  CHECK ((scope_kind = 'global' AND scope_id IS NULL) OR (scope_kind != 'global'))
+);
+
+CREATE TABLE IF NOT EXISTS entity_links (
+  id TEXT PRIMARY KEY,
+  from_entity_id TEXT NOT NULL,
+  to_entity_id TEXT NOT NULL,
+  relation_type TEXT NOT NULL,
+  confidence REAL NOT NULL DEFAULT 0.5,
+  source_basis TEXT NOT NULL,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (from_entity_id) REFERENCES entity_nodes(id) ON DELETE CASCADE,
+  FOREIGN KEY (to_entity_id) REFERENCES entity_nodes(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS entity_timeline_events (
+  id TEXT PRIMARY KEY,
+  entity_id TEXT NOT NULL,
+  event_type TEXT NOT NULL,
+  valid_from INTEGER,
+  valid_to INTEGER,
+  observed_at INTEGER,
+  source_ref TEXT,
+  summary TEXT NOT NULL,
+  details TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (entity_id) REFERENCES entity_nodes(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_entity_nodes_kind ON entity_nodes(kind);
+CREATE INDEX IF NOT EXISTS idx_entity_nodes_preferred_label ON entity_nodes(preferred_label);
+CREATE INDEX IF NOT EXISTS idx_entity_nodes_scope ON entity_nodes(scope_kind, scope_id);
+
+CREATE INDEX IF NOT EXISTS idx_entity_aliases_entity_id ON entity_aliases(entity_id);
+CREATE INDEX IF NOT EXISTS idx_entity_aliases_normalized_label ON entity_aliases(normalized_label);
+
+CREATE INDEX IF NOT EXISTS idx_entity_observations_normalized_label ON entity_observations(normalized_form);
+CREATE INDEX IF NOT EXISTS idx_entity_observations_observed_at ON entity_observations(timestamp_observed);
+CREATE INDEX IF NOT EXISTS idx_entity_observations_scope ON entity_observations(scope_kind, scope_id);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_observations_source_record
+  ON entity_observations(source_connector, source_raw_record_id);
+
+CREATE INDEX IF NOT EXISTS idx_entity_links_from ON entity_links(from_entity_id);
+CREATE INDEX IF NOT EXISTS idx_entity_links_to ON entity_links(to_entity_id);
+
+CREATE INDEX IF NOT EXISTS idx_entity_timeline_events_entity_id ON entity_timeline_events(entity_id);
+CREATE INDEX IF NOT EXISTS idx_entity_timeline_events_observed_at ON entity_timeline_events(observed_at);
+
+INSERT OR IGNORE INTO schema_version (version, description)
+VALUES (26, 'Create canonical entity core tables');

package/db/migrations/027-create-entity-review-support.sql

@@ -0,0 +1,44 @@
+CREATE TABLE IF NOT EXISTS entity_resolution_candidates (
+  id TEXT PRIMARY KEY,
+  candidate_kind TEXT NOT NULL CHECK (candidate_kind IN ('alias_to_entity', 'entity_to_entity', 'cluster')),
+  left_ref TEXT NOT NULL,
+  right_ref TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('pending', 'auto_merged', 'approved', 'rejected', 'deferred')),
+  score_total REAL NOT NULL DEFAULT 0,
+  score_structural REAL NOT NULL DEFAULT 0,
+  score_string REAL NOT NULL DEFAULT 0,
+  score_context REAL NOT NULL DEFAULT 0,
+  score_graph REAL NOT NULL DEFAULT 0,
+  score_embedding REAL NOT NULL DEFAULT 0,
+  rule_trace TEXT,
+  extractor_version TEXT NOT NULL,
+  embedding_model_version TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000)
+);
+
+CREATE TABLE IF NOT EXISTS entity_merge_actions (
+  id TEXT PRIMARY KEY,
+  action_type TEXT NOT NULL CHECK (action_type IN ('merge', 'reject', 'defer', 'split')),
+  source_entity_id TEXT,
+  target_entity_id TEXT,
+  candidate_id TEXT,
+  actor_type TEXT NOT NULL CHECK (actor_type IN ('system', 'user', 'agent')),
+  actor_id TEXT,
+  reason TEXT NOT NULL,
+  evidence_json TEXT NOT NULL,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (source_entity_id) REFERENCES entity_nodes(id) ON DELETE SET NULL,
+  FOREIGN KEY (target_entity_id) REFERENCES entity_nodes(id) ON DELETE SET NULL,
+  FOREIGN KEY (candidate_id) REFERENCES entity_resolution_candidates(id) ON DELETE SET NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_entity_candidates_status_score
+  ON entity_resolution_candidates(status, score_total DESC, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_entity_merge_actions_candidate_id
+  ON entity_merge_actions(candidate_id);
+CREATE INDEX IF NOT EXISTS idx_entity_merge_actions_source_target
+  ON entity_merge_actions(source_entity_id, target_entity_id);
+
+INSERT OR IGNORE INTO schema_version (version, description)
+VALUES (27, 'Create canonical entity review support tables');

package/db/migrations/028-create-entity-audit-tables.sql

@@ -0,0 +1,45 @@
+CREATE TABLE IF NOT EXISTS entity_audit_runs (
+  id TEXT PRIMARY KEY,
+  status TEXT NOT NULL CHECK (status IN ('running', 'complete', 'failed', 'timeout')),
+  baseline_run_id TEXT,
+  classification TEXT CHECK (classification IN ('improved', 'stable', 'regressed', 'inconclusive')),
+  metric_summary_json TEXT,
+  reason TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  completed_at INTEGER,
+  FOREIGN KEY (baseline_run_id) REFERENCES entity_audit_runs(id) ON DELETE SET NULL
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS entity_audit_runs_single_running
+  ON entity_audit_runs(status)
+  WHERE status = 'running';
+
+CREATE INDEX IF NOT EXISTS idx_entity_audit_runs_created_at
+  ON entity_audit_runs(created_at DESC);
+
+CREATE TABLE IF NOT EXISTS entity_audit_metrics (
+  run_id TEXT NOT NULL,
+  metric_name TEXT NOT NULL,
+  metric_value REAL NOT NULL,
+  metric_meta_json TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  PRIMARY KEY (run_id, metric_name),
+  FOREIGN KEY (run_id) REFERENCES entity_audit_runs(id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS entity_audit_findings (
+  id TEXT PRIMARY KEY,
+  run_id TEXT NOT NULL,
+  finding_kind TEXT NOT NULL,
+  severity TEXT NOT NULL CHECK (severity IN ('info', 'warn', 'critical')),
+  summary TEXT NOT NULL,
+  details_json TEXT,
+  created_at INTEGER NOT NULL DEFAULT (unixepoch() * 1000),
+  FOREIGN KEY (run_id) REFERENCES entity_audit_runs(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_entity_audit_findings_run_id
+  ON entity_audit_findings(run_id);
+
+INSERT OR IGNORE INTO schema_version (version, description)
+VALUES (28, 'Create entity benchmark and audit run storage');

package/db/migrations/029-extend-entity-observations-observation-type.sql

@@ -0,0 +1,60 @@
+ALTER TABLE entity_observations
+  ADD COLUMN observation_type TEXT NOT NULL DEFAULT 'generic'
+  CHECK (observation_type IN ('generic', 'author', 'channel'));
+
+UPDATE entity_observations
+SET source_raw_db_ref = ''
+WHERE source_raw_db_ref IS NULL;
+
+DROP INDEX IF EXISTS idx_entity_observations_source_record;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_observations_source_record
+  ON entity_observations(
+    source_connector,
+    source_raw_db_ref,
+    source_raw_record_id,
+    observation_type
+  );
+
+CREATE TRIGGER IF NOT EXISTS trg_entity_nodes_scope_guard_insert
+BEFORE INSERT ON entity_nodes
+WHEN (
+  (NEW.scope_kind = 'global' AND NEW.scope_id IS NOT NULL)
+  OR (NEW.scope_kind IS NOT NULL AND NEW.scope_kind != 'global' AND NEW.scope_id IS NULL)
+)
+BEGIN
+  SELECT RAISE(ABORT, 'entity_nodes scope_id invalid for scope_kind');
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_entity_nodes_scope_guard_update
+BEFORE UPDATE ON entity_nodes
+WHEN (
+  (NEW.scope_kind = 'global' AND NEW.scope_id IS NOT NULL)
+  OR (NEW.scope_kind IS NOT NULL AND NEW.scope_kind != 'global' AND NEW.scope_id IS NULL)
+)
+BEGIN
+  SELECT RAISE(ABORT, 'entity_nodes scope_id invalid for scope_kind');
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_entity_observations_scope_guard_insert
+BEFORE INSERT ON entity_observations
+WHEN (
+  (NEW.scope_kind = 'global' AND NEW.scope_id IS NOT NULL)
+  OR (NEW.scope_kind IS NOT NULL AND NEW.scope_kind != 'global' AND NEW.scope_id IS NULL)
+)
+BEGIN
+  SELECT RAISE(ABORT, 'entity_observations scope_id invalid for scope_kind');
+END;
+
+CREATE TRIGGER IF NOT EXISTS trg_entity_observations_scope_guard_update
+BEFORE UPDATE ON entity_observations
+WHEN (
+  (NEW.scope_kind = 'global' AND NEW.scope_id IS NOT NULL)
+  OR (NEW.scope_kind IS NOT NULL AND NEW.scope_kind != 'global' AND NEW.scope_id IS NULL)
+)
+BEGIN
+  SELECT RAISE(ABORT, 'entity_observations scope_id invalid for scope_kind');
+END;
+
+INSERT OR IGNORE INTO schema_version (version, description)
+VALUES (29, 'Extend entity observations with observation type and scope guards');