@pentatonic-ai/ai-agent-sdk 0.9.6 → 0.10.0

package/packages/memory-engine-v2/org-model/migrations/001_init.sql

@@ -0,0 +1,390 @@
+-- pentatonic-memory-engine v2: org-model schema, initial migration.
+--
+-- Design principles encoded in this schema:
+--
+-- 1. Append-only event log + materialised views. The `events` table is
+--    the source of truth; `entities`, `facts`, `relationships` are
+--    materialised views over it. To iterate the schema, add columns;
+--    never alter existing ones. To rebuild the view, drop + replay.
+--
+-- 2. Provenance + participant-set + disclosure-class on every fact
+--    from day one. Even if enforcement (the privacy plane / Layer-P)
+--    is later, the columns exist. Retrofitting these into a populated
+--    multi-tenant schema = the schema-altering operation we explicitly
+--    can't tolerate. See PR #285 (extraction-objectives.md) for the
+--    semantic contract these columns enforce.
+--
+-- 3. Forget is a deletion, not an orphan mark. When an evidence
+--    record is forgotten, every fact whose entire provenance set
+--    is forgotten gets deleted in the same transaction. Orphan-
+--    marking a derivative of erased data is itself GDPR non-
+--    compliance — see privacy model L4 / fix #8.
+--
+-- 4. Content-hash IDs everywhere. Idempotent upsert is the only
+--    write contract. Re-ingestion converges instead of inflating
+--    (the v1 L0 840k vs 594k bloat).
+
+-- ----------------------------------------------------------------------
+-- Enums
+-- ----------------------------------------------------------------------
+
+-- Source category, set at ingest time by extractor-sync's per-source
+-- rules. Drives downstream routing (typed retrieval) + extraction
+-- prompts. v1 hardcoded these to 'general' for every record; v2 makes
+-- it required.
+CREATE TYPE source_kind AS ENUM (
+  'chat',         -- slack, teams, dm
+  'note',         -- gmail, drafts
+  'doc',          -- drive, notion, confluence
+  'event',        -- calendar meetings
+  'ticket',       -- jira, linear, github issues
+  'commit',       -- github commits / PRs
+  'system',       -- system-of-record snapshots (CRM, ERP)
+  'agent'         -- agent-generated content (chat_turn from pip)
+);
+
+-- Internal vs external classification. "Internal" = the writing user's
+-- organisation; "external" = counterparty / customer / partner. Drives
+-- the disclosure-class invariant in PR #280 (counterparty-taint spike).
+CREATE TYPE participant_kind AS ENUM (
+  'internal',
+  'external',
+  'mixed',        -- a thread with both
+  'unknown'       -- can't classify; treated as 'external' for safety
+);
+
+-- Disclosure class — how widely can a fact be re-surfaced? Default
+-- 'private' until proven otherwise. Drives cross-user inference taint.
+CREATE TYPE disclosure_class AS ENUM (
+  'public',       -- explicitly shared (e.g., docs marked external)
+  'team',         -- visible across user's org but not externally
+  'private',      -- visible only to participant set
+  'restricted'    -- legal hold / contains PII / etc.
+);
+
+-- Extraction stage. Provisional = extracted by deterministic
+-- fast-path (extractor-sync). Distilled = upgraded by LLM
+-- (extractor-async). Verified = human-confirmed.
+CREATE TYPE extraction_stage AS ENUM (
+  'provisional',
+  'distilled',
+  'verified'
+);
+
+-- ----------------------------------------------------------------------
+-- events — the append-only source of truth.
+--
+-- Every STORE_MEMORY / FORGET_MEMORY event the engine processes lands
+-- here. The materialised views below are derived from this table; if
+-- they're ever wrong, drop and replay.
+-- ----------------------------------------------------------------------
+
+CREATE TABLE events (
+  -- Content-addressed primary key: sha256(arena || ':' || canonical_content).
+  -- Idempotent: re-emit the same event, same ID, no-op insert.
+  id                  TEXT PRIMARY KEY,
+
+  -- Arena scope. 'tenant' = clientId, 'user' = clientId:userId.
+  arena               TEXT NOT NULL,
+  client_id           TEXT NOT NULL,
+  user_id             TEXT,
+
+  -- Event semantics.
+  event_type          TEXT NOT NULL,  -- STORE_MEMORY | FORGET_MEMORY | CHAT_TURN
+  source_kind         source_kind NOT NULL,
+  source_id           TEXT,           -- producer's stable id (msg ts, drive id, etc.)
+
+  -- Content + canonical hash. content_hash is what dedups across
+  -- re-ingests; content is the raw payload (lazy-materialisable —
+  -- could move to object storage later if RAM matters).
+  content             TEXT NOT NULL,
+  content_hash        TEXT NOT NULL,
+
+  -- Provenance (mandatory from day one — see header note 2).
+  participant_set     TEXT[] NOT NULL,           -- arena IDs that can see
+  participant_kind    participant_kind NOT NULL DEFAULT 'unknown',
+  disclosure_class    disclosure_class NOT NULL DEFAULT 'private',
+
+  -- Producer-supplied metadata bag. Untyped to avoid lock-in; the
+  -- extractor reads from here per source_kind.
+  attributes          JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+  -- Timestamps. emitted_at is producer-supplied; received_at is when
+  -- we landed it. Both useful for forensic queries.
+  emitted_at          TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  received_at         TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  -- FORGET_MEMORY events reference the event they're erasing.
+  forgets             TEXT REFERENCES events(id) ON DELETE SET NULL,
+
+  CONSTRAINT valid_event_type CHECK (event_type IN ('STORE_MEMORY', 'FORGET_MEMORY', 'CHAT_TURN'))
+);
+
+CREATE INDEX idx_events_arena ON events(arena);
+CREATE INDEX idx_events_content_hash ON events(content_hash);
+CREATE INDEX idx_events_source_kind ON events(source_kind);
+CREATE INDEX idx_events_emitted_at ON events(emitted_at DESC);
+CREATE INDEX idx_events_client_user ON events(client_id, user_id);
+CREATE INDEX idx_events_participant_set ON events USING GIN(participant_set);
+CREATE INDEX idx_events_attributes ON events USING GIN(attributes jsonb_path_ops);
+
+-- ----------------------------------------------------------------------
+-- entities — extracted nouns, people, projects, etc.
+--
+-- Many-to-one with events: one event can mention multiple entities.
+-- entity_id is content-derived (lowercased + stemmed, plus
+-- participant_set for tenant isolation) so re-extractions converge.
+-- ----------------------------------------------------------------------
+
+CREATE TABLE entities (
+  id                  TEXT PRIMARY KEY,
+  arena               TEXT NOT NULL,
+  entity_type         TEXT NOT NULL,             -- person | org | project | concept | ...
+  canonical_name      TEXT NOT NULL,
+  -- Surface forms used to derive this canonical (for fuzzy matching).
+  -- Append-only — never remove.
+  aliases             TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+  -- Provenance: which events first mentioned this entity.
+  provenance_event_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
+
+  participant_set     TEXT[] NOT NULL,
+  disclosure_class    disclosure_class NOT NULL DEFAULT 'private',
+
+  attributes          JSONB NOT NULL DEFAULT '{}'::jsonb,
+  first_seen          TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  last_seen           TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX idx_entities_arena ON entities(arena);
+CREATE INDEX idx_entities_entity_type ON entities(entity_type);
+CREATE INDEX idx_entities_canonical_name ON entities(canonical_name);
+CREATE INDEX idx_entities_aliases ON entities USING GIN(aliases);
+CREATE INDEX idx_entities_provenance ON entities USING GIN(provenance_event_ids);
+
+-- ----------------------------------------------------------------------
+-- facts — extracted decisions, commitments, statements.
+--
+-- This is the org-model proper: structured knowledge derived from
+-- events. Each fact cites the events it was extracted from
+-- (provenance_event_ids). When all citing events are forgotten, the
+-- fact gets deleted (see header note 3).
+-- ----------------------------------------------------------------------
+
+CREATE TABLE facts (
+  id                  TEXT PRIMARY KEY,
+  arena               TEXT NOT NULL,
+  -- Fact category, drives prompt selection in extractor-async and
+  -- typed routing in compat's read path.
+  category            TEXT NOT NULL,             -- decision | commitment | state | mention | observation
+  subject_entity_id   TEXT REFERENCES entities(id) ON DELETE SET NULL,
+  predicate           TEXT,                       -- 'committed_to' | 'decided' | 'mentioned_in' etc.
+  object_entity_id    TEXT REFERENCES entities(id) ON DELETE SET NULL,
+  -- Free-text natural-language form. Useful for LLM context-building
+  -- and for showing the user.
+  statement           TEXT NOT NULL,
+
+  -- Provenance. Citing events that support this fact. If all are
+  -- deleted via FORGET_MEMORY cascade, this fact gets deleted too.
+  provenance_event_ids TEXT[] NOT NULL,
+  -- Extraction confidence + stage.
+  stage               extraction_stage NOT NULL DEFAULT 'provisional',
+  confidence          REAL NOT NULL DEFAULT 0.5 CHECK (confidence BETWEEN 0.0 AND 1.0),
+
+  -- Provenance privacy from day one.
+  participant_set     TEXT[] NOT NULL,
+  disclosure_class    disclosure_class NOT NULL DEFAULT 'private',
+
+  -- Temporal anchoring.
+  asserted_at         TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  -- For commitments / deadlines: when the fact resolves or expires.
+  effective_until     TIMESTAMPTZ,
+
+  attributes          JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+CREATE INDEX idx_facts_arena ON facts(arena);
+CREATE INDEX idx_facts_category ON facts(category);
+CREATE INDEX idx_facts_subject ON facts(subject_entity_id);
+CREATE INDEX idx_facts_object ON facts(object_entity_id);
+CREATE INDEX idx_facts_provenance ON facts USING GIN(provenance_event_ids);
+CREATE INDEX idx_facts_stage ON facts(stage);
+CREATE INDEX idx_facts_asserted_at ON facts(asserted_at DESC);
+
+-- ----------------------------------------------------------------------
+-- relationships — edges between entities.
+--
+-- pgGraph / recursive-CTE substrate. Lets us compute personFacets,
+-- peopleList, and spreading-activation queries over the canonical
+-- store, retiring Neo4j (pending the spike in #278).
+-- ----------------------------------------------------------------------
+
+CREATE TABLE relationships (
+  id                  TEXT PRIMARY KEY,
+  arena               TEXT NOT NULL,
+  from_entity_id      TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+  to_entity_id        TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+  relationship_type   TEXT NOT NULL,             -- communicated_with | works_for | mentions | etc.
+
+  -- Edge weight (for spreading-activation). Aggregated from supporting
+  -- events; updated in-place by the extractor (idempotent: see content_hash).
+  weight              REAL NOT NULL DEFAULT 1.0,
+  first_seen          TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  last_seen           TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  provenance_event_ids TEXT[] NOT NULL,
+  participant_set     TEXT[] NOT NULL,
+  disclosure_class    disclosure_class NOT NULL DEFAULT 'private',
+
+  attributes          JSONB NOT NULL DEFAULT '{}'::jsonb,
+
+  -- Idempotent edge identity: (arena, from, to, type) is unique. Lets
+  -- re-extractions update weight + last_seen without duplicating.
+  UNIQUE(arena, from_entity_id, to_entity_id, relationship_type)
+);
+
+CREATE INDEX idx_relationships_arena ON relationships(arena);
+CREATE INDEX idx_relationships_from ON relationships(from_entity_id);
+CREATE INDEX idx_relationships_to ON relationships(to_entity_id);
+CREATE INDEX idx_relationships_type ON relationships(relationship_type);
+CREATE INDEX idx_relationships_provenance ON relationships USING GIN(provenance_event_ids);
+
+-- ----------------------------------------------------------------------
+-- distillation_queue — work queue for the async LLM extractor.
+--
+-- extractor-sync enqueues an entry for every event it processes;
+-- extractor-async polls + claims items, runs the LLM, writes facts
+-- back, marks the item done. Crash-safe: claim_expires_at lets a
+-- stuck-mid-extraction item re-surface.
+-- ----------------------------------------------------------------------
+
+CREATE TABLE distillation_queue (
+  id                  BIGSERIAL PRIMARY KEY,
+  event_id            TEXT NOT NULL REFERENCES events(id) ON DELETE CASCADE,
+  enqueued_at         TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  -- Worker that's currently processing this. NULL = unclaimed.
+  claimed_by          TEXT,
+  claimed_at          TIMESTAMPTZ,
+  -- Claims expire so a crashed worker doesn't strand items.
+  claim_expires_at    TIMESTAMPTZ,
+  -- Status: pending | claimed | done | failed.
+  status              TEXT NOT NULL DEFAULT 'pending',
+  attempts            INT NOT NULL DEFAULT 0,
+  last_error          TEXT,
+  completed_at        TIMESTAMPTZ,
+
+  CONSTRAINT valid_status CHECK (status IN ('pending', 'claimed', 'done', 'failed'))
+);
+
+CREATE INDEX idx_distillation_status ON distillation_queue(status);
+CREATE INDEX idx_distillation_event_id ON distillation_queue(event_id);
+CREATE INDEX idx_distillation_claim_expires ON distillation_queue(claim_expires_at)
+  WHERE status = 'claimed';
+
+-- ----------------------------------------------------------------------
+-- vector_provenance — links from vector-index payloads back to events.
+--
+-- The evidence index (Qdrant) stores embeddings + lightweight payload;
+-- the canonical record is here. compat queries vector-index → uses
+-- payload.event_id → fetches event from this table → projects facts
+-- citing that event. Late materialisation in action.
+--
+-- Why a table and not just relying on Qdrant payload: it lets us
+-- enforce FK cascade on event delete (the orphan-prevention contract)
+-- in the same transaction as the event delete. The vector-index
+-- payload becomes the "soft pointer"; this table is the hard
+-- transactional link.
+-- ----------------------------------------------------------------------
+
+CREATE TABLE vector_provenance (
+  -- vector_id = Qdrant point ID for this embedding.
+  vector_id           TEXT PRIMARY KEY,
+  event_id            TEXT NOT NULL REFERENCES events(id) ON DELETE CASCADE,
+  chunk_index         INT NOT NULL DEFAULT 0,    -- for multi-chunk events
+  embedded_at         TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  embedding_model     TEXT NOT NULL,             -- 'nv-embed-v2' etc.
+  embedding_dim       INT NOT NULL               -- 4096 today
+);
+
+CREATE INDEX idx_vector_provenance_event_id ON vector_provenance(event_id);
+
+-- ----------------------------------------------------------------------
+-- audit_events — operator + system events (forensic + observability).
+--
+-- Mirrors the v1 audit_events table for compatibility. compat writes
+-- here on every /store, /search, /forget call so existing dashboards
+-- + queries keep working.
+-- ----------------------------------------------------------------------
+
+CREATE TABLE audit_events (
+  id                  BIGSERIAL PRIMARY KEY,
+  client_id           TEXT NOT NULL,
+  operation           TEXT NOT NULL,             -- store | search | forget | etc.
+  actor_user_id       TEXT,
+  request_hash        TEXT NOT NULL,
+  engine_id           TEXT,                       -- predicted/actual record id
+  layer_id            TEXT,
+  duration_ms         INT NOT NULL,
+  status              TEXT NOT NULL,             -- queued | ok | engine_error | ...
+  error_msg           TEXT,
+  record_count        INT,
+  created_at          TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX idx_audit_events_client_id ON audit_events(client_id);
+CREATE INDEX idx_audit_events_operation ON audit_events(operation);
+CREATE INDEX idx_audit_events_created_at ON audit_events(created_at DESC);
+
+-- ----------------------------------------------------------------------
+-- forget cascade trigger.
+--
+-- When a FORGET_MEMORY event lands, the events row referenced by
+-- `forgets` gets deleted. CASCADE deletes vector_provenance + drops
+-- the event_id from any provenance_event_ids array in facts /
+-- relationships / entities. If a fact's provenance_event_ids becomes
+-- empty as a result, the fact gets deleted in the same trigger.
+--
+-- This is the "FORGET is a deletion, not an orphan mark" contract
+-- from the reconciliation doc.
+-- ----------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION cascade_forget() RETURNS TRIGGER AS $$
+BEGIN
+  -- Remove the forgotten event_id from every fact's provenance.
+  UPDATE facts SET provenance_event_ids = array_remove(provenance_event_ids, OLD.id)
+    WHERE OLD.id = ANY(provenance_event_ids);
+  -- Delete facts whose entire provenance is now gone.
+  DELETE FROM facts WHERE provenance_event_ids = ARRAY[]::TEXT[];
+
+  -- Same treatment for relationships.
+  UPDATE relationships SET provenance_event_ids = array_remove(provenance_event_ids, OLD.id)
+    WHERE OLD.id = ANY(provenance_event_ids);
+  DELETE FROM relationships WHERE provenance_event_ids = ARRAY[]::TEXT[];
+
+  -- Same for entities. (Entities are usually multi-event; this only
+  -- fires for one-event entities, which is fine.)
+  UPDATE entities SET provenance_event_ids = array_remove(provenance_event_ids, OLD.id)
+    WHERE OLD.id = ANY(provenance_event_ids);
+  DELETE FROM entities WHERE provenance_event_ids = ARRAY[]::TEXT[];
+
+  RETURN OLD;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER trigger_cascade_forget
+  BEFORE DELETE ON events
+  FOR EACH ROW EXECUTE FUNCTION cascade_forget();
+
+-- ----------------------------------------------------------------------
+-- Health check view — used by compat /health/deep.
+-- ----------------------------------------------------------------------
+
+CREATE OR REPLACE VIEW health_counts AS
+  SELECT
+    (SELECT COUNT(*) FROM events) AS events_count,
+    (SELECT COUNT(*) FROM entities) AS entities_count,
+    (SELECT COUNT(*) FROM facts) AS facts_count,
+    (SELECT COUNT(*) FROM relationships) AS relationships_count,
+    (SELECT COUNT(*) FROM vector_provenance) AS vector_records_count,
+    (SELECT COUNT(*) FROM distillation_queue WHERE status = 'pending') AS distillation_pending,
+    (SELECT COUNT(*) FROM distillation_queue WHERE status = 'claimed') AS distillation_in_flight,
+    (SELECT COUNT(*) FROM distillation_queue WHERE status = 'failed') AS distillation_failed;