@lobu/cli 6.0.1 → 7.0.0

package/dist/db/migrations/20260427140000_identity_engine_indexes.sql

@@ -0,0 +1,140 @@
+-- migrate:up
+
+-- Identity-engine schema additions.
+--
+-- The follow-up identity engine writes connector-emitted facts as rows in
+-- `events` with `semantic_type='identity_fact'` and stores derivation
+-- provenance as metadata on `entity_relationships`. Auto-create rules live
+-- as JSONB on `entity_relationship_types.metadata` (compiled from YAML by
+-- the seeder). All three shapes need selective indexes so the hot paths
+-- don't full-scan.
+--
+-- Pattern matches the existing per-namespace event metadata indexes added
+-- in 20260419120000_add_event_identity_indexes.sql.
+
+-- ── Rule storage on relationship types ─────────────────────────────────
+-- The engine reads each relationship type's `metadata.autoCreateWhen[]` to
+-- decide which rules to fire on each incoming fact. Adding the column up
+-- front (NULL allowed) keeps the seeder change non-destructive.
+ALTER TABLE public.entity_relationship_types
+    ADD COLUMN IF NOT EXISTS metadata jsonb;
+
+CREATE INDEX IF NOT EXISTS idx_entity_relationship_types_has_auto_create
+    ON public.entity_relationship_types ((metadata->'autoCreateWhen'))
+    WHERE metadata ? 'autoCreateWhen' AND deleted_at IS NULL;
+
+-- ── Identity lookup ─────────────────────────────────────────────────────
+-- "Find the entity in catalog X whose normalized identity value matches
+-- this fact's normalizedValue." Composite expression index keyed on the
+-- (org, namespace, normalizedValue) tuple. Partial: only fact-typed events
+-- get indexed, so total size scales with fact volume (small) not event
+-- volume (huge).
+CREATE INDEX IF NOT EXISTS idx_events_identity_fact_lookup
+    ON public.events (
+        organization_id,
+        (metadata->>'namespace'),
+        (metadata->>'normalizedValue')
+    )
+    WHERE semantic_type = 'identity_fact';
+
+-- ── Per-account fact diff ───────────────────────────────────────────────
+-- "Find every active fact for this provider account." Used by the engine
+-- to diff prior facts vs current set on refresh — drops fall out of the
+-- result and get superseded. Keyed on `providerStableId` (not
+-- `sourceAccountId`): BetterAuth may issue a fresh account row on
+-- reconnect; we want facts to chain across that boundary.
+CREATE INDEX IF NOT EXISTS idx_events_identity_fact_account
+    ON public.events (
+        connector_key,
+        (metadata->>'providerStableId'),
+        (metadata->>'namespace')
+    )
+    WHERE semantic_type = 'identity_fact';
+
+-- ── Provenance reverse-lookup ───────────────────────────────────────────
+-- "Find every relationship derived from this fact event." Used at
+-- revocation: when a fact is superseded, find auto-created relationships
+-- that referenced its event_id and revoke them.
+CREATE INDEX IF NOT EXISTS idx_entity_relationships_derived_from_event
+    ON public.entity_relationships (
+        ((metadata->'derivedFrom'->>'sourceEventId'))
+    )
+    WHERE metadata ? 'derivedFrom';
+
+-- ── Rule-version drift detection ────────────────────────────────────────
+-- "Find every relationship derived from this rule type at this version."
+-- Used by reconcile when a rule changes — find derivations stamped with
+-- an older version, revoke or refresh them.
+CREATE INDEX IF NOT EXISTS idx_entity_relationships_derived_from_rule
+    ON public.entity_relationships (
+        ((metadata->'derivedFrom'->>'relationshipTypeId')),
+        ((metadata->'derivedFrom'->>'ruleVersion'))
+    )
+    WHERE metadata ? 'derivedFrom';
+
+-- ── Idempotent derivation insert ────────────────────────────────────────
+-- Pi P0.3 — even with the engine's advisory lock, this partial unique
+-- constraint catches accidental double-inserts (e.g. when a different code
+-- path tries to write the same auto-derived edge). ON CONFLICT DO NOTHING
+-- in the engine relies on this index to fire.
+--
+-- Existing installs may already have duplicate live triples. Collapse those
+-- first so adding the invariant is safe and deterministic.
+WITH duplicate_live_triples AS (
+    SELECT id,
+           ROW_NUMBER() OVER (
+               PARTITION BY from_entity_id, to_entity_id, relationship_type_id
+               ORDER BY created_at ASC NULLS LAST, id ASC
+           ) AS rn
+    FROM public.entity_relationships
+    WHERE deleted_at IS NULL
+)
+UPDATE public.entity_relationships r
+SET deleted_at = NOW(), updated_at = NOW()
+FROM duplicate_live_triples d
+WHERE r.id = d.id
+  AND d.rn > 1;
+
+CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_relationships_live_triple
+    ON public.entity_relationships (from_entity_id, to_entity_id, relationship_type_id)
+    WHERE deleted_at IS NULL;
+
+-- ── Per-namespace entity metadata indexes ───────────────────────────────
+-- Pi P1.10 — the engine's findEntitiesByMetadataField does
+-- `WHERE metadata->>$field = $value`. Without per-field expression indexes
+-- this is a sequential scan on the entities table for every fact it
+-- processes. Add the common identity namespaces here; future namespaces
+-- get an entry in this same list when their first rule lands.
+--
+-- `idx_entities_metadata_domain` already exists in the baseline schema.
+
+CREATE INDEX IF NOT EXISTS idx_entities_metadata_email
+    ON public.entities ((metadata->>'email'), organization_id)
+    WHERE (metadata->>'email') IS NOT NULL AND deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_entities_metadata_linkedin_url
+    ON public.entities ((metadata->>'linkedin_url'), organization_id)
+    WHERE (metadata->>'linkedin_url') IS NOT NULL AND deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_entities_metadata_github_handle
+    ON public.entities ((metadata->>'github_handle'), organization_id)
+    WHERE (metadata->>'github_handle') IS NOT NULL AND deleted_at IS NULL;
+
+CREATE INDEX IF NOT EXISTS idx_entities_metadata_twitter_handle
+    ON public.entities ((metadata->>'twitter_handle'), organization_id)
+    WHERE (metadata->>'twitter_handle') IS NOT NULL AND deleted_at IS NULL;
+
+
+-- migrate:down
+
+DROP INDEX IF EXISTS public.idx_entities_metadata_twitter_handle;
+DROP INDEX IF EXISTS public.idx_entities_metadata_github_handle;
+DROP INDEX IF EXISTS public.idx_entities_metadata_linkedin_url;
+DROP INDEX IF EXISTS public.idx_entities_metadata_email;
+DROP INDEX IF EXISTS public.idx_entity_relationships_live_triple;
+DROP INDEX IF EXISTS public.idx_entity_relationships_derived_from_rule;
+DROP INDEX IF EXISTS public.idx_entity_relationships_derived_from_event;
+DROP INDEX IF EXISTS public.idx_events_identity_fact_account;
+DROP INDEX IF EXISTS public.idx_events_identity_fact_lookup;
+DROP INDEX IF EXISTS public.idx_entity_relationship_types_has_auto_create;
+ALTER TABLE public.entity_relationship_types DROP COLUMN IF EXISTS metadata;

package/dist/db/migrations/20260427150000_drop_events_source_id.sql

@@ -0,0 +1,177 @@
+-- migrate:up transaction:false
+
+-- `events.source_id` is legacy and duplicates `events.connection_id`. New code
+-- no longer writes or reads it, and scoring now partitions by connection_id.
+-- This migration intentionally runs after that code is deployed.
+SET lock_timeout = '5s';
+
+DROP VIEW IF EXISTS public.event_thread_tree;
+DROP VIEW IF EXISTS public.current_event_records;
+
+DROP INDEX IF EXISTS public.idx_event_length;
+DROP INDEX IF EXISTS public.idx_event_source_id;
+
+ALTER TABLE public.events
+    DROP COLUMN IF EXISTS source_id;
+
+DROP TRIGGER IF EXISTS normalize_event_created_by ON public.events;
+DROP FUNCTION IF EXISTS public.normalize_event_created_by();
+
+CREATE VIEW public.current_event_records AS
+ SELECT e.id,
+    e.organization_id,
+    e.entity_ids,
+    e.origin_id,
+    e.title,
+    e.payload_type,
+    e.payload_text,
+    e.payload_data,
+    e.payload_template,
+    e.attachments,
+    e.metadata,
+    e.score,
+    emb.embedding,
+    e.author_name,
+    e.source_url,
+    e.occurred_at,
+    e.created_at,
+    e.origin_parent_id,
+    COALESCE(length(e.payload_text), 0) AS content_length,
+    e.origin_type,
+    e.connector_key,
+    e.connection_id,
+    e.feed_key,
+    e.feed_id,
+    e.run_id,
+    e.semantic_type,
+    e.client_id,
+    e.created_by,
+    e.interaction_type,
+    e.interaction_status,
+    e.interaction_input_schema,
+    e.interaction_input,
+    e.interaction_output,
+    e.interaction_error,
+    e.supersedes_event_id
+   FROM (public.events e
+     LEFT JOIN public.event_embeddings emb ON ((emb.event_id = e.id)))
+  WHERE (NOT (EXISTS ( SELECT 1
+           FROM public.events newer
+          WHERE (newer.supersedes_event_id = e.id))));
+
+CREATE VIEW public.event_thread_tree AS
+ SELECT e.id,
+    e.origin_id,
+    e.origin_parent_id,
+    e.occurred_at,
+    COALESCE(parent.origin_id, e.origin_id) AS root_origin_id,
+    COALESCE(parent.occurred_at, e.occurred_at) AS root_occurred_at,
+    COALESCE(parent.score, e.score) AS root_score,
+        CASE
+            WHEN (e.origin_parent_id IS NULL) THEN 0
+            ELSE 1
+        END AS depth,
+    ARRAY[(COALESCE(parent.occurred_at, e.occurred_at))::text, (e.id)::text] AS sort_path
+   FROM (public.current_event_records e
+     LEFT JOIN public.current_event_records parent ON (((e.origin_parent_id = parent.origin_id) AND (e.entity_ids && parent.entity_ids))));
+
+-- migrate:down transaction:false
+
+SET lock_timeout = '5s';
+
+DROP VIEW IF EXISTS public.event_thread_tree;
+DROP VIEW IF EXISTS public.current_event_records;
+
+ALTER TABLE public.events
+    ADD COLUMN IF NOT EXISTS source_id integer;
+
+UPDATE public.events
+   SET source_id = connection_id::integer
+ WHERE source_id IS NULL
+   AND connection_id IS NOT NULL
+   AND connection_id <= 2147483647;
+
+CREATE INDEX IF NOT EXISTS idx_event_length
+    ON public.events (source_id, (COALESCE(length(payload_text), 0)));
+
+CREATE INDEX IF NOT EXISTS idx_event_source_id
+    ON public.events (source_id);
+
+CREATE OR REPLACE FUNCTION public.normalize_event_created_by()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+    IF NEW.created_by IN ('system', 'api') AND NOT EXISTS (
+        SELECT 1 FROM public."user" u WHERE u.id = NEW.created_by
+    ) THEN
+        NEW.created_by := NULL;
+    END IF;
+    RETURN NEW;
+END;
+$$;
+
+DROP TRIGGER IF EXISTS normalize_event_created_by ON public.events;
+CREATE TRIGGER normalize_event_created_by
+    BEFORE INSERT OR UPDATE OF created_by ON public.events
+    FOR EACH ROW
+    EXECUTE FUNCTION public.normalize_event_created_by();
+
+CREATE VIEW public.current_event_records AS
+ SELECT e.id,
+    e.organization_id,
+    e.entity_ids,
+    e.source_id,
+    e.origin_id,
+    e.title,
+    e.payload_type,
+    e.payload_text,
+    e.payload_data,
+    e.payload_template,
+    e.attachments,
+    e.metadata,
+    e.score,
+    emb.embedding,
+    e.author_name,
+    e.source_url,
+    e.occurred_at,
+    e.created_at,
+    e.origin_parent_id,
+    COALESCE(length(e.payload_text), 0) AS content_length,
+    e.origin_type,
+    e.connector_key,
+    e.connection_id,
+    e.feed_key,
+    e.feed_id,
+    e.run_id,
+    e.semantic_type,
+    e.client_id,
+    e.created_by,
+    e.interaction_type,
+    e.interaction_status,
+    e.interaction_input_schema,
+    e.interaction_input,
+    e.interaction_output,
+    e.interaction_error,
+    e.supersedes_event_id
+   FROM (public.events e
+     LEFT JOIN public.event_embeddings emb ON ((emb.event_id = e.id)))
+  WHERE (NOT (EXISTS ( SELECT 1
+           FROM public.events newer
+          WHERE (newer.supersedes_event_id = e.id))));
+
+CREATE VIEW public.event_thread_tree AS
+ SELECT e.id,
+    e.origin_id,
+    e.origin_parent_id,
+    e.occurred_at,
+    COALESCE(parent.origin_id, e.origin_id) AS root_origin_id,
+    COALESCE(parent.occurred_at, e.occurred_at) AS root_occurred_at,
+    COALESCE(parent.score, e.score) AS root_score,
+        CASE
+            WHEN (e.origin_parent_id IS NULL) THEN 0
+            ELSE 1
+        END AS depth,
+    ARRAY[(COALESCE(parent.occurred_at, e.occurred_at))::text, (e.id)::text] AS sort_path
+   FROM (public.current_event_records e
+     LEFT JOIN public.current_event_records parent ON (((e.origin_parent_id = parent.origin_id) AND (e.entity_ids && parent.entity_ids))));

package/dist/db/migrations/20260427160000_drop_dead_schema.sql

@@ -0,0 +1,76 @@
+-- migrate:up
+
+-- Drop schema objects that have no readers, writers, or references in the
+-- TypeScript source tree (`packages/server/src` is the source of
+-- truth — `lobu-cli/runtime` is a generated mirror).
+--
+-- Verification:
+--   * Strict SQL grep across packages/{server,gateway,worker,
+--     web,connector-sdk}/src for FROM/JOIN/INSERT/UPDATE/REFERENCES.
+--   * `query_sql` allowlist (utils/table-schema.ts) does not list any of
+--     the dropped tables, columns, or the view.
+--   * better-auth `teams` is not enabled in src/auth/index.tsx, so the
+--     `team` table and `member.teamId` are unused at runtime.
+--   * `verification` is intentionally NOT dropped — better-auth's
+--     magicLink and phoneNumber plugins write OTPs/tokens there.
+--
+-- Idempotent (DROP IF EXISTS) so it is safe whether the object exists or
+-- was already removed out-of-band.
+
+-- ---------------------------------------------------------------------------
+-- Tables: empty in prod, no code paths reference them
+-- ---------------------------------------------------------------------------
+
+DROP TABLE IF EXISTS public.rate_limits;
+DROP TABLE IF EXISTS public.workers;
+DROP TABLE IF EXISTS public.source_type_auth_defaults;
+DROP TABLE IF EXISTS public.workspace_settings;
+
+-- `team` is referenced by `member.teamId`. Drop the inbound FK + column
+-- first so the table drop does not depend on CASCADE behavior.
+ALTER TABLE public.member DROP CONSTRAINT IF EXISTS "member_teamId_fkey";
+ALTER TABLE public.member DROP COLUMN IF EXISTS "teamId";
+DROP TABLE IF EXISTS public.team;
+
+-- ---------------------------------------------------------------------------
+-- Views: defined in baseline, not referenced anywhere in code
+-- ---------------------------------------------------------------------------
+
+DROP VIEW IF EXISTS public.event_thread_tree;
+
+-- ---------------------------------------------------------------------------
+-- Columns: always-NULL in prod, never read or written by any code path
+-- ---------------------------------------------------------------------------
+
+ALTER TABLE public.watchers
+  DROP COLUMN IF EXISTS lobu_schedule_id,
+  DROP COLUMN IF EXISTS registry_ref,
+  DROP COLUMN IF EXISTS registry_repo,
+  DROP COLUMN IF EXISTS registry_type;
+
+-- watcher_versions.source_path is unused (the heavily-used `source_path`
+-- column lives on connector_versions; same name, different table).
+ALTER TABLE public.watcher_versions
+  DROP COLUMN IF EXISTS source_path,
+  DROP COLUMN IF EXISTS source_repository,
+  DROP COLUMN IF EXISTS source_ref,
+  DROP COLUMN IF EXISTS source_commit_sha,
+  DROP COLUMN IF EXISTS sources_schema;
+
+ALTER TABLE public.connector_versions
+  DROP COLUMN IF EXISTS source_repository,
+  DROP COLUMN IF EXISTS source_ref,
+  DROP COLUMN IF EXISTS source_commit_sha;
+
+ALTER TABLE public.event_embeddings
+  DROP COLUMN IF EXISTS model_key;
+
+
+-- migrate:down
+
+-- This cleanup is one-way. Re-creating these objects would require copying
+-- DDL from the baseline migration (00000000000000_baseline.sql) and the
+-- column definitions inferred from prior schema dumps. Down is intentionally
+-- a no-op — recovery is via `git revert` of this migration plus manual DDL,
+-- not via dbmate rollback.
+SELECT 1;