@aperdomoll90/ledger-ai 1.4.0 → 1.4.2

package/dist/migrations/migrations/005-audit-log.sql

@@ -0,0 +1,22 @@
+-- Migration 005: Audit Log Table
+-- Phase 1 of v2 roadmap
+-- Append-only log of every write operation for rollback, sync, rate limiting, and observability
+--
+-- Design decisions:
+--   - No FK on note_id (audit entries must survive note deletion)
+--   - JSONB diff column stores old values for rollback
+--   - Indexes on note_id (lookup by note), created_at (time-range), domain (filter by domain)
+
+CREATE TABLE IF NOT EXISTS audit_log (
+  id         bigserial    PRIMARY KEY,
+  note_id    bigint,
+  domain     text,
+  operation  text         NOT NULL,
+  agent      text         NOT NULL,
+  diff       jsonb,
+  created_at timestamptz  DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_audit_note_id ON audit_log (note_id);
+CREATE INDEX IF NOT EXISTS idx_audit_created ON audit_log (created_at);
+CREATE INDEX IF NOT EXISTS idx_audit_domain  ON audit_log (domain);

package/dist/migrations/migrations/005_opportunities.sql

@@ -0,0 +1,48 @@
+-- 005_opportunities.sql
+-- Hunter agent: opportunity tracking for freelance and employment leads
+
+CREATE TABLE opportunities (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  -- Source
+  url TEXT UNIQUE NOT NULL,
+  title TEXT NOT NULL,
+  source TEXT NOT NULL,
+  raw_description TEXT,
+
+  -- Classification
+  track TEXT NOT NULL CHECK (track IN ('freelance', 'employment')),
+
+  -- Scoring (nullable — raw listings stored before scoring)
+  score INTEGER,
+  score_breakdown JSONB,
+  recommendation TEXT CHECK (recommendation IN ('apply', 'ask', 'skip')),
+  reject_reasons TEXT[],
+
+  -- AI Output (nullable — populated after scoring)
+  summary TEXT,
+  key_requirements TEXT[],
+  compensation_range TEXT,
+  scored_at TIMESTAMPTZ,
+
+  -- Pipeline
+  status TEXT NOT NULL DEFAULT 'new' CHECK (status IN (
+    'new', 'reviewed', 'applied', 'interviewing', 'won', 'lost', 'rejected'
+  )),
+  status_changed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  notes TEXT
+);
+
+-- Indexes
+CREATE INDEX idx_opportunities_track ON opportunities(track);
+CREATE INDEX idx_opportunities_status ON opportunities(status);
+CREATE INDEX idx_opportunities_score ON opportunities(score DESC);
+CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC);
+CREATE INDEX idx_opportunities_unscored ON opportunities(created_at) WHERE score IS NULL;
+
+-- Auto-update updated_at trigger (reuse pattern from notes table)
+CREATE TRIGGER update_opportunities_updated_at
+  BEFORE UPDATE ON opportunities
+  FOR EACH ROW EXECUTE FUNCTION update_updated_at();

package/dist/migrations/migrations/006-audited-operations.sql

@@ -0,0 +1,235 @@
+-- Migration 006: Transactional Note Operations with Audit
+-- Every note write (create, update, delete) is wrapped in a transaction with its audit entry.
+-- If either the note write or the audit entry fails, both roll back.
+-- This prevents: orphaned notes without audit trails, audit entries for failed writes,
+-- and partial chunk writes.
+
+-- =============================================================================
+-- 1. CREATE: Insert one or more note rows + audit entry in one transaction
+-- =============================================================================
+-- Accepts arrays for multi-chunk support. Single-chunk = arrays of length 1.
+-- Returns the IDs of all inserted rows.
+
+CREATE OR REPLACE FUNCTION note_create(
+  p_contents     text[],
+  p_metadatas    jsonb[],
+  p_embeddings   vector(1536)[],
+  p_ids          bigint[]       DEFAULT NULL,   -- optional: preserve specific IDs (for upsert)
+  p_created_at   timestamptz    DEFAULT NULL     -- optional: preserve original timestamp
+) RETURNS bigint[] AS $$
+DECLARE
+  v_ids       bigint[] := '{}';
+  v_id        bigint;
+  v_ts        timestamptz := COALESCE(p_created_at, now());
+  i           int;
+BEGIN
+  FOR i IN 1..array_length(p_contents, 1) LOOP
+    IF p_ids IS NOT NULL AND p_ids[i] IS NOT NULL THEN
+      INSERT INTO notes (id, content, metadata, embedding, created_at)
+      VALUES (p_ids[i], p_contents[i], p_metadatas[i], p_embeddings[i], v_ts)
+      RETURNING id INTO v_id;
+    ELSE
+      INSERT INTO notes (content, metadata, embedding, created_at)
+      VALUES (p_contents[i], p_metadatas[i], p_embeddings[i], v_ts)
+      RETURNING id INTO v_id;
+    END IF;
+    v_ids := v_ids || v_id;
+  END LOOP;
+
+  -- Audit: log creation using first chunk's metadata for domain/agent
+  INSERT INTO audit_log (note_id, domain, operation, agent, diff)
+  VALUES (
+    v_ids[1],
+    p_metadatas[1]->>'domain',
+    'create',
+    COALESCE(p_metadatas[1]->>'agent', 'unknown'),
+    NULL
+  );
+
+  RETURN v_ids;
+END;
+$$ LANGUAGE plpgsql;
+
+-- =============================================================================
+-- 2. UPDATE: Update a single note in-place + audit entry
+-- =============================================================================
+-- Reads old content/metadata before overwriting for the audit diff.
+
+CREATE OR REPLACE FUNCTION note_update(
+  p_id         bigint,
+  p_content    text,
+  p_metadata   jsonb,
+  p_embedding  vector(1536)
+) RETURNS void AS $$
+DECLARE
+  v_old_content  text;
+  v_old_metadata jsonb;
+BEGIN
+  -- Read old values for audit diff
+  SELECT content, metadata INTO v_old_content, v_old_metadata
+  FROM notes WHERE id = p_id;
+
+  IF NOT FOUND THEN
+    RAISE EXCEPTION 'Note % not found', p_id;
+  END IF;
+
+  -- Update the note
+  UPDATE notes
+  SET content = p_content, metadata = p_metadata, embedding = p_embedding, updated_at = now()
+  WHERE id = p_id;
+
+  -- Audit: log update with old values for rollback
+  INSERT INTO audit_log (note_id, domain, operation, agent, diff)
+  VALUES (
+    p_id,
+    p_metadata->>'domain',
+    'update',
+    COALESCE(p_metadata->>'agent', 'unknown'),
+    jsonb_build_object('content', v_old_content, 'metadata', v_old_metadata)
+  );
+END;
+$$ LANGUAGE plpgsql;
+
+-- =============================================================================
+-- 3. REPLACE: Delete old note/chunks + insert new note/chunks + audit
+-- =============================================================================
+-- For chunk-count changes: atomically removes old content and inserts new.
+-- Preserves the original note ID for the first new chunk.
+
+CREATE OR REPLACE FUNCTION note_replace(
+  p_old_id          bigint,           -- original note ID to preserve
+  p_old_chunk_group text,             -- NULL if old was single-chunk
+  p_contents        text[],
+  p_metadatas       jsonb[],
+  p_embeddings      vector(1536)[],
+  p_created_at      timestamptz
+) RETURNS bigint[] AS $$
+DECLARE
+  v_old_content  text;
+  v_old_metadata jsonb;
+  v_ids          bigint[] := '{}';
+  v_id           bigint;
+  i              int;
+BEGIN
+  -- Read old values for audit diff (from the original note)
+  SELECT content, metadata INTO v_old_content, v_old_metadata
+  FROM notes WHERE id = p_old_id;
+
+  -- Delete old note/chunks
+  IF p_old_chunk_group IS NOT NULL THEN
+    DELETE FROM notes WHERE metadata->>'chunk_group' = p_old_chunk_group;
+  ELSE
+    DELETE FROM notes WHERE id = p_old_id;
+  END IF;
+
+  -- Insert new note/chunks
+  FOR i IN 1..array_length(p_contents, 1) LOOP
+    IF i = 1 THEN
+      -- First chunk preserves the original ID
+      INSERT INTO notes (id, content, metadata, embedding, created_at)
+      VALUES (p_old_id, p_contents[i], p_metadatas[i], p_embeddings[i], p_created_at)
+      RETURNING id INTO v_id;
+    ELSE
+      INSERT INTO notes (content, metadata, embedding, created_at)
+      VALUES (p_contents[i], p_metadatas[i], p_embeddings[i], p_created_at)
+      RETURNING id INTO v_id;
+    END IF;
+    v_ids := v_ids || v_id;
+  END LOOP;
+
+  -- Audit: log as update with old values for rollback
+  INSERT INTO audit_log (note_id, domain, operation, agent, diff)
+  VALUES (
+    p_old_id,
+    p_metadatas[1]->>'domain',
+    'update',
+    COALESCE(p_metadatas[1]->>'agent', 'unknown'),
+    jsonb_build_object('content', v_old_content, 'metadata', v_old_metadata)
+  );
+
+  RETURN v_ids;
+END;
+$$ LANGUAGE plpgsql;
+
+-- =============================================================================
+-- 4. DELETE: Delete note/chunks + audit entry with full content for rollback
+-- =============================================================================
+
+CREATE OR REPLACE FUNCTION note_delete(
+  p_id           bigint,
+  p_chunk_group  text,       -- NULL if single-chunk note
+  p_agent        text
+) RETURNS void AS $$
+DECLARE
+  v_content  text;
+  v_metadata jsonb;
+BEGIN
+  -- Read full content for rollback
+  SELECT content, metadata INTO v_content, v_metadata
+  FROM notes WHERE id = p_id;
+
+  IF NOT FOUND THEN
+    RAISE EXCEPTION 'Note % not found', p_id;
+  END IF;
+
+  -- Audit FIRST: store full content before deletion
+  INSERT INTO audit_log (note_id, domain, operation, agent, diff)
+  VALUES (
+    p_id,
+    v_metadata->>'domain',
+    'delete',
+    p_agent,
+    jsonb_build_object('content', v_content, 'metadata', v_metadata)
+  );
+
+  -- Then delete
+  IF p_chunk_group IS NOT NULL THEN
+    DELETE FROM notes WHERE metadata->>'chunk_group' = p_chunk_group;
+  ELSE
+    DELETE FROM notes WHERE id = p_id;
+  END IF;
+END;
+$$ LANGUAGE plpgsql;
+
+-- =============================================================================
+-- 5. UPDATE METADATA: Update metadata fields + audit with changed fields
+-- =============================================================================
+
+CREATE OR REPLACE FUNCTION note_update_metadata(
+  p_id          bigint,
+  p_metadata    jsonb       -- fields to merge (not full replacement)
+) RETURNS void AS $$
+DECLARE
+  v_old_metadata jsonb;
+  v_merged       jsonb;
+  v_changed      jsonb;
+BEGIN
+  SELECT metadata INTO v_old_metadata FROM notes WHERE id = p_id;
+
+  IF NOT FOUND THEN
+    RAISE EXCEPTION 'Note % not found', p_id;
+  END IF;
+
+  -- Merge: new fields override old, old fields preserved
+  v_merged := v_old_metadata || p_metadata;
+
+  -- Calculate which fields actually changed (old values of changed keys)
+  SELECT COALESCE(jsonb_object_agg(key, v_old_metadata->key), '{}'::jsonb)
+  INTO v_changed
+  FROM jsonb_each(p_metadata)
+  WHERE v_old_metadata->key IS DISTINCT FROM p_metadata->key;
+
+  -- Update
+  UPDATE notes SET metadata = v_merged, updated_at = now() WHERE id = p_id;
+
+  -- Audit
+  INSERT INTO audit_log (note_id, domain, operation, agent, diff)
+  VALUES (
+    p_id,
+    v_merged->>'domain',
+    'update_metadata',
+    COALESCE(v_merged->>'agent', 'unknown'),
+    jsonb_build_object('metadata', v_changed)
+  );
+END;
+$$ LANGUAGE plpgsql;

package/dist/migrations/migrations/006_hunt_analytics.sql

@@ -0,0 +1,38 @@
+-- 006_hunt_analytics.sql
+-- Hunter agent: periodic analytics summaries
+
+CREATE TABLE hunt_analytics (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  period_start DATE NOT NULL,
+  period_end DATE NOT NULL,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+
+  -- Volume
+  total_found INTEGER NOT NULL,
+  freelance_count INTEGER NOT NULL,
+  employment_count INTEGER NOT NULL,
+
+  -- Score Distribution
+  score_80_plus INTEGER NOT NULL,
+  score_60_79 INTEGER NOT NULL,
+  score_40_59 INTEGER NOT NULL,
+  score_below_40 INTEGER NOT NULL,
+
+  -- Pipeline
+  applied_count INTEGER NOT NULL,
+  won_count INTEGER NOT NULL,
+  lost_count INTEGER NOT NULL,
+  apply_rate NUMERIC(5,2),
+  win_rate NUMERIC(5,2),
+
+  -- Patterns (JSONB)
+  top_reject_reasons JSONB,
+  top_skills_demanded JSONB,
+  avg_score_by_track JSONB,
+  compensation_ranges JSONB,
+
+  -- Insights
+  summary TEXT
+);
+
+CREATE UNIQUE INDEX idx_hunt_analytics_period ON hunt_analytics(period_start, period_end);

package/dist/migrations/migrations/007-eval-golden-judgments.sql

@@ -0,0 +1,119 @@
+-- ============================================================
+-- Migration 007 — Phase 4.6.2: Graded Relevance
+--
+-- Adds eval_golden_judgments table + 3 RPC functions for atomic writes.
+-- Does NOT drop eval_golden_dataset.expected_doc_ids (deferred to migration 008).
+-- ============================================================
+
+-- ============================================================
+-- Table
+-- ============================================================
+CREATE TABLE IF NOT EXISTS eval_golden_judgments (
+  id            bigserial    PRIMARY KEY,
+  golden_id     bigint       NOT NULL REFERENCES eval_golden_dataset(id) ON DELETE CASCADE,
+  document_id   bigint       NOT NULL REFERENCES documents(id)           ON DELETE CASCADE,
+  grade         smallint     NOT NULL CHECK (grade BETWEEN 0 AND 3),
+  judged_at     timestamptz  NOT NULL DEFAULT now(),
+  judged_by     text         NOT NULL DEFAULT 'adrian',
+  notes         text,
+  UNIQUE (golden_id, document_id)
+);
+
+-- ============================================================
+-- Indexes
+-- ============================================================
+CREATE INDEX IF NOT EXISTS idx_golden_judgments_golden_id   ON eval_golden_judgments(golden_id);
+CREATE INDEX IF NOT EXISTS idx_golden_judgments_document_id ON eval_golden_judgments(document_id);
+CREATE INDEX IF NOT EXISTS idx_golden_judgments_grade       ON eval_golden_judgments(grade);
+
+-- ============================================================
+-- Row-level security
+-- ============================================================
+ALTER TABLE eval_golden_judgments ENABLE ROW LEVEL SECURITY;
+
+DROP POLICY IF EXISTS eval_golden_judgments_service_all ON eval_golden_judgments;
+CREATE POLICY eval_golden_judgments_service_all
+  ON eval_golden_judgments
+  FOR ALL
+  TO service_role
+  USING (true)
+  WITH CHECK (true);
+
+-- ============================================================
+-- RPC: judgment_create
+-- Inserts one judgment. Errors on duplicate (golden_id, document_id).
+-- Returns the new row id.
+-- ============================================================
+CREATE OR REPLACE FUNCTION judgment_create(
+  p_golden_id    bigint,
+  p_document_id  bigint,
+  p_grade        smallint,
+  p_judged_by    text DEFAULT 'adrian',
+  p_notes        text DEFAULT NULL
+) RETURNS bigint
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  v_id bigint;
+BEGIN
+  INSERT INTO eval_golden_judgments (golden_id, document_id, grade, judged_by, notes)
+  VALUES (p_golden_id, p_document_id, p_grade, p_judged_by, p_notes)
+  RETURNING id INTO v_id;
+  RETURN v_id;
+END;
+$$;
+
+-- ============================================================
+-- RPC: judgment_update
+-- Updates grade (and optionally notes) for an existing judgment.
+-- Bumps judged_at. Errors if no row matches.
+-- ============================================================
+CREATE OR REPLACE FUNCTION judgment_update(
+  p_golden_id    bigint,
+  p_document_id  bigint,
+  p_grade        smallint,
+  p_notes        text DEFAULT NULL
+) RETURNS void
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  v_updated int;
+BEGIN
+  UPDATE eval_golden_judgments
+     SET grade     = p_grade,
+         notes     = COALESCE(p_notes, notes),
+         judged_at = now()
+   WHERE golden_id   = p_golden_id
+     AND document_id = p_document_id;
+
+  GET DIAGNOSTICS v_updated = ROW_COUNT;
+  IF v_updated = 0 THEN
+    RAISE EXCEPTION 'judgment_update: no judgment exists for (golden_id=%, document_id=%)',
+      p_golden_id, p_document_id;
+  END IF;
+END;
+$$;
+
+-- ============================================================
+-- RPC: judgment_delete
+-- Removes a judgment. Idempotent on missing row (no error).
+-- ============================================================
+CREATE OR REPLACE FUNCTION judgment_delete(
+  p_golden_id    bigint,
+  p_document_id  bigint
+) RETURNS void
+LANGUAGE plpgsql
+AS $$
+BEGIN
+  DELETE FROM eval_golden_judgments
+   WHERE golden_id   = p_golden_id
+     AND document_id = p_document_id;
+END;
+$$;
+
+-- ============================================================
+-- Grants
+-- ============================================================
+GRANT EXECUTE ON FUNCTION judgment_create(bigint, bigint, smallint, text, text) TO service_role;
+GRANT EXECUTE ON FUNCTION judgment_update(bigint, bigint, smallint, text)       TO service_role;
+GRANT EXECUTE ON FUNCTION judgment_delete(bigint, bigint)                       TO service_role;

package/dist/migrations/migrations/008-drop-expected-doc-ids.sql

@@ -0,0 +1,9 @@
+-- ============================================================
+-- Migration 008b — Phase 4.6.2 cleanup: drop legacy column
+--
+-- Runs AFTER the graded dataset is fully populated and run 14
+-- has landed. The expected_doc_ids data was converted to grade-3
+-- rows in eval_golden_judgments by convert-judgments-to-graded.ts.
+-- ============================================================
+
+ALTER TABLE eval_golden_dataset DROP COLUMN IF EXISTS expected_doc_ids;

package/dist/migrations/migrations/008-judge-helpers.sql

@@ -0,0 +1,21 @@
+-- ============================================================
+-- Migration 008 — Phase 4.6.2: judge helpers
+--
+-- Adds the count_golden_with_min_judgments() function used by the
+-- ledger eval:judge CLI progress display.
+-- ============================================================
+
+CREATE OR REPLACE FUNCTION count_golden_with_min_judgments(p_min int)
+RETURNS bigint
+LANGUAGE sql
+STABLE
+AS $$
+  SELECT count(*)::bigint FROM (
+    SELECT golden_id
+    FROM eval_golden_judgments
+    GROUP BY golden_id
+    HAVING count(*) >= p_min
+  ) AS qualifying;
+$$;
+
+GRANT EXECUTE ON FUNCTION count_golden_with_min_judgments(int) TO service_role;