@intelligo-dev/core 1.0.0-beta.1 → 1.0.0-beta.14

package/src/db/migrations/0001_reconcile_chain_built_databases.sql

@@ -0,0 +1,53 @@
+-- A database built by the framework's pre-1.0 migration chain is adopted
+-- at 0000_baseline without running it. That chain never quite matched
+-- the schema the code was written against — every environment used to
+-- be provisioned with `db:push` — so this brings such a database to the
+-- baseline's shape. On a database created from the baseline every
+-- statement is a no-op.
+
+-- conversations.metadata was created as text; the code reads and writes
+-- it as jsonb.
+DO $$
+BEGIN
+	IF (SELECT data_type FROM information_schema.columns
+	    WHERE table_schema = 'public' AND table_name = 'conversations' AND column_name = 'metadata') = 'text' THEN
+		ALTER TABLE "conversations" ALTER COLUMN "metadata" TYPE jsonb USING NULLIF("metadata", '')::jsonb;
+	END IF;
+END $$;
+--> statement-breakpoint
+-- Deleting a user keeps the invitations they sent, without an inviter.
+ALTER TABLE "invitation" ALTER COLUMN "inviter_id" DROP NOT NULL;
+--> statement-breakpoint
+DO $$
+BEGIN
+	IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'invitation_inviter_id_users_id_fk' AND confdeltype <> 'n') THEN
+		ALTER TABLE "invitation" DROP CONSTRAINT "invitation_inviter_id_users_id_fk";
+		ALTER TABLE "invitation" ADD CONSTRAINT "invitation_inviter_id_users_id_fk"
+			FOREIGN KEY ("inviter_id") REFERENCES "users"("id") ON DELETE SET NULL;
+	END IF;
+END $$;
+--> statement-breakpoint
+-- Deleting a workspace keeps its finance events for the books.
+DO $$
+BEGIN
+	IF EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'finance_events_workspace_id_organization_id_fk' AND confdeltype <> 'n') THEN
+		ALTER TABLE "finance_events" DROP CONSTRAINT "finance_events_workspace_id_organization_id_fk";
+		ALTER TABLE "finance_events" ADD CONSTRAINT "finance_events_workspace_id_organization_id_fk"
+			FOREIGN KEY ("workspace_id") REFERENCES "organization"("id") ON DELETE SET NULL;
+	END IF;
+END $$;
+--> statement-breakpoint
+-- One quota row per user per workspace. The old chain made this a
+-- unique index of the same name, which serves ON CONFLICT equally.
+DO $$
+BEGIN
+	IF to_regclass('public.user_quotas_user_workspace_unique') IS NULL THEN
+		ALTER TABLE "user_quotas" ADD CONSTRAINT "user_quotas_user_workspace_unique" UNIQUE ("user_id", "workspace_id");
+	END IF;
+END $$;
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "conversations_workspace_user_idx" ON "conversations" USING btree ("workspace_id", "user_id");
+--> statement-breakpoint
+CREATE INDEX IF NOT EXISTS "documents_user_id_idx" ON "documents" USING btree ("user_id");
+--> statement-breakpoint
+CREATE UNIQUE INDEX IF NOT EXISTS "member_organization_user_uniq" ON "member" USING btree ("organization_id", "user_id");

package/src/db/migrations/0002_webhook_claim_lease.sql

@@ -0,0 +1 @@
+ALTER TABLE "finance_events" ADD COLUMN "claimed_at" timestamp;

package/src/db/migrations/0003_local_payments.sql

@@ -0,0 +1,18 @@
+CREATE TABLE "payments" (
+	"id" text PRIMARY KEY NOT NULL,
+	"provider" text NOT NULL,
+	"invoice_id" text NOT NULL,
+	"workspace_id" text NOT NULL,
+	"user_id" text,
+	"reference" text NOT NULL,
+	"amount_minor" integer NOT NULL,
+	"currency" text NOT NULL,
+	"status" text DEFAULT 'pending' NOT NULL,
+	"fulfilled_at" timestamp,
+	"created_at" timestamp DEFAULT now() NOT NULL,
+	CONSTRAINT "payments_invoice_id_unique" UNIQUE("invoice_id")
+);
+--> statement-breakpoint
+ALTER TABLE "payments" ADD CONSTRAINT "payments_workspace_id_organization_id_fk" FOREIGN KEY ("workspace_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "payments" ADD CONSTRAINT "payments_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "payments_workspace_id_idx" ON "payments" USING btree ("workspace_id");

package/src/db/migrations/README.md

@@ -1,43 +1,111 @@
 # Migrations
 
-`drizzle-kit migrate` applies the `.sql` files listed in
-`meta/_journal.json`, in journal order. Both halves have to agree, and
-for most of this repository's life they did not: the journal stopped at
-entry 11 while the directory grew to 39 files, so `migrate` silently
-applied a quarter of the chain and every environment was really
-provisioned by `drizzle-kit push` from `schema.ts`.
-
-That is how `0032` shipped documenting a unique constraint on a
-`minute_bucket` column no migration ever created — the rate limiter's
-`ON CONFLICT` could not have resolved on a database built from these
-files. `0037` repairs it, and `intelligo doctor` now fails CI if the
-journal and the directory drift apart again.
+The framework's schema is one migration, `0000_baseline.sql`, generated
+from the Drizzle schema (`src/db/schema/*` plus the audit, executions
+and jobs packages' `schema.ts`) with the pieces Drizzle cannot express
+added by hand: the `vector` extension, the HNSW index on memory
+embeddings, and the triggers that keep `audit_events` and
+`user_memory_audit` append-only. Later migrations follow it in
+`meta/_journal.json`.
+
+A migration here changes structure only. It never inserts or edits rows:
+what a deployment bills in, its plans and its agents come from its own
+composition root (`ensureBillingSettingsRow`, the plan registry).
+`tests/architecture/baseline.test.ts` enforces it.
+
+## In a consumer application
+
+This directory ships inside the published package (it is in `files`,
+next to `dist`), and `intelligo migrate` applies it: it resolves the
+chain from `node_modules/@intelligo-dev/core/src/db/migrations`, applies
+what is pending by content hash, and records it in drizzle's default
+`drizzle.__drizzle_migrations` table — the table
+`intelligo migrate --check` reads.
+
+The tables an application owns are a second chain, kept apart on
+purpose: drizzle-kit applies by timestamp, so a framework migration
+published after the consumer generated one of theirs would be skipped
+silently if the two shared a journal. `intelligo create` scaffolds a
+`drizzle.config.ts` whose `schema` lists only the consumer's files,
+whose `out` is the app's own `./drizzle`, and whose `migrations.table`
+is `__app_migrations`. A consumer schema file may `references()` a
+framework table — drizzle-kit emits the foreign key by name and does
+not try to create the referenced table. The scaffold's scripts:
+
+```
+pnpm db:generate   # drizzle-kit generate           — your chain
+pnpm db:migrate    # intelligo migrate && drizzle-kit migrate
+pnpm db:check      # intelligo migrate --check      — deploy gate
+```
+
+`drizzle-kit push` has no place in this layout: it diffs the whole
+database against the schema it can see, and the consumer config sees
+only the consumer's tables.
 
 ## Adding a migration
 
-Prefer `pnpm --filter @intelligo-dev/core db:generate`, which writes both
-the `.sql` file and its journal entry. A hand-written file needs its
-tag added to `meta/_journal.json` by hand — `doctor` will tell you if
-you forget.
+Change the schema, then `pnpm --filter @intelligo-dev/core db:generate`,
+which writes the `.sql` file, its snapshot and its journal entry. A
+hand-written file (a trigger, a data-free `DO` block) needs its journal
+entry added by hand — `intelligo doctor` fails if the journal and the
+directory disagree.
+
+## Databases from before 1.0
+
+Before 1.0 the framework's schema was a chain of 48 migrations that had
+grown with the product it came from; some created tables the framework
+no longer owns. `legacy-chain.json` lists that chain's tags and content
+hashes — not its SQL. A database that ran the whole chain is **adopted**:
+`intelligo migrate` records `0000_baseline` as applied without running
+it, then applies `0001_reconcile_chain_built_databases`, which brings
+the few places where the old chain and the schema disagreed into line
+(a no-op on a database created from the baseline). Tables the old chain
+created that the framework no longer defines are left untouched; a
+product that still uses them keeps them in its own migrations.
 
-Note that `db:generate` diffs against `meta/NNNN_snapshot.json`, and
-snapshots exist only up to `0011`. Until the snapshots are rebuilt,
-generate produces a full-schema diff rather than an incremental one;
-hand-write the file and add the journal entry instead.
+A database that ran only part of the old chain is refused, and
+`intelligo migrate --check` names the migrations it has not run
+(`legacyMissing` in `--json`). No published version finishes the old
+chain: `@intelligo-dev/core@1.0.0-beta.6`, the last one on npm before the
+baseline, ships it through `0042_sessions_active_organization_id`, and
+the versions that carried `0043_attachments`, `0044_money_micros`,
+`0045_drop_legacy_money_columns` and `0046_user_quotas_per_workspace`
+never reached npm. Their SQL is not in the public repository either:
+its last revision before the baseline, `fdc6c3d`, lists all 48 in the
+journal but carries no `.sql` files. That leaves three ways forward;
+try each on a restored copy first.
 
-## Baselining an existing database
+1. **You have the missing SQL** — a source checkout of the pre-1.0
+   framework that contains the files. Apply each missing migration in
+   chain order and record it in `drizzle.__drizzle_migrations`: `hash`
+   is the sha256 of the file's contents, which must be the one
+   `legacy-chain.json` lists, and `created_at` its journal `when`. Once
+   the database holds the whole chain, `intelligo migrate` adopts it.
+2. **You do not** — bring the schema to the baseline by hand, then
+   record the baseline as applied the way a push-provisioned database is
+   (next section). Create an empty database, run `intelligo migrate`
+   there, and compare the two schemas (`pg_dump --schema-only`); what
+   differs is what `0043`–`0046` would have done. Mind the data, not
+   only the columns: `0044` backfilled every money amount into
+   `*_micros` columns (MNT at ×1,000,000) with a currency, and `0045`
+   dropped the old `*_mnt` and cents columns only after that copy.
+   Clear the old chain's rows from `drizzle.__drizzle_migrations` and
+   insert the current chain's before the first `intelligo migrate`.
+3. **Start over** — create the database from the baseline and move the
+   rows you keep across. For a deployment whose data is disposable this
+   is the shortest path.
 
-Because the journal previously listed only 12 of the 39 files, a
-database provisioned with `push` has the schema but no record of the
-migrations. Running `migrate` against it now would try to apply all 39.
-Most are `IF NOT EXISTS`-guarded, but not all.
+## Baselining a push-provisioned database
 
-Mark them as already applied before the first `migrate` on such a
-database:
+A database provisioned with `drizzle-kit push` has the schema but no
+migration records, and `intelligo migrate` refuses it rather than
+applying the baseline to tables that already exist. Record the chain as
+applied before the first `migrate`, one row per journal entry, in
+journal order: `hash` is the sha256 of the `.sql` file's contents and
+`created_at` the entry's `when`.
 
 ```sql
--- One row per journal entry, in journal order. hash is the sha256 of
--- the .sql file contents; drizzle compares it on subsequent runs.
+CREATE SCHEMA IF NOT EXISTS drizzle;
 CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
   id SERIAL PRIMARY KEY,
   hash text NOT NULL,
@@ -45,13 +113,12 @@ CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (
 );
 ```
 
-Generate the rows with the file hashes and the `when` values from the
-journal, then verify with `intelligo doctor` and a `migrate` run that
-reports nothing to apply. Do this on a restored copy first.
+Then verify with `intelligo migrate --check`. Do this on a restored copy
+first.
 
 ## Verifying the chain
 
-CI replays every file against an empty database on each run (see the
-`e2e` job) and then asserts the columns the running code depends on.
-That check is what proves these files are a provisioning mechanism
-rather than documentation.
+CI replays every file against an empty database on each run and then
+asserts the columns the running code depends on, and the database
+integration suites run against that replay — which is what proves these
+files are a provisioning mechanism rather than documentation.

package/src/db/migrations/legacy-chain.json

@@ -0,0 +1,206 @@
+{
+  "$comment": "The framework's migration chain before the 1.0 baseline: every entry's tag and the sha256 of its .sql file, as drizzle recorded it in drizzle.__drizzle_migrations. `intelligo migrate` reads it to adopt a database that ran the whole old chain \u2014 it records 0000_baseline as applied without running it. The SQL itself is not shipped. `alternates`: other hashes the same migration was recorded under \u2014 the SQL npm shipped in 1.0.0-beta.6 differs from the repository's for three of them.",
+  "entries": [
+    {
+      "tag": "0000_sharp_night_nurse",
+      "hash": "095685bd97c0d51b9ec5413d866c84e1ff1f5a86c51a1228ec1a87d372f19899"
+    },
+    {
+      "tag": "0001_hesitant_vapor",
+      "hash": "403a33243ec4687a3c114a7d1b60e469988452fb54aea93d2e69c29839c86f04"
+    },
+    {
+      "tag": "0002_numerous_lady_vermin",
+      "hash": "90d931beaaea073d5a363318119449c957cde32282dd53e5bcd5ad488174f97a"
+    },
+    {
+      "tag": "0003_mighty_jean_grey",
+      "hash": "9d0128a08223a0a112ac8e197315e85eb86d9d7a92028577bc763a5dd9ac84ec"
+    },
+    {
+      "tag": "0004_cynical_grandmaster",
+      "hash": "53ca240bb183a4416bac27fe430019e36b5c63a8e592f33a8d058f986ad56d97"
+    },
+    {
+      "tag": "0005_careful_switch",
+      "hash": "b1bd2845c02008f37d49f4c89ab56f8584ec4c3a1f95f23291e836d21bec9465"
+    },
+    {
+      "tag": "0006_tense_blue_marvel",
+      "hash": "2bab2a70e4ce5fda000d595193800b07ff88a8492da95034c79e82da4b937755"
+    },
+    {
+      "tag": "0007_next_squadron_sinister",
+      "hash": "c071b97e8edd8b034ecda51764e40ca9e50b3719103f31acd098cfc2e9390b98"
+    },
+    {
+      "tag": "0008_powerful_scream",
+      "hash": "f88f2a55354442843293abf91069bf0f0a274e8426a954930cba06ed89c557c6"
+    },
+    {
+      "tag": "0009_sad_ink",
+      "hash": "c5b957dd7abf7dedbe13d2fb4a6c4f3ab5da26d3b8690f8e4f610c53b9366c85"
+    },
+    {
+      "tag": "0010_shocking_omega_flight",
+      "hash": "c82b4be9f37fbf4d65c0713231a05f162f87a2713df39956155ded4e228217f3"
+    },
+    {
+      "tag": "0011_ambiguous_otto_octavius",
+      "hash": "0688a1389e8eb66a645103ec3984e77859d91889359cb75eb36b1cd4a07052e5"
+    },
+    {
+      "tag": "0011_hnsw_vector_index",
+      "hash": "69663f644a3df6c644c5b8b2cd5edbf6045d8c6a319dc15fad35e413128ef243"
+    },
+    {
+      "tag": "0012_user_profiles",
+      "hash": "aec846995de9798d97524dbfc2bbfc65249cf8142f35c80746688d2ce1807fd5"
+    },
+    {
+      "tag": "0013_trending_seasonal_columns",
+      "hash": "895d018d85eee1dec5f24ef2c00b5f58de8d036577a8fefec87fef785605891c"
+    },
+    {
+      "tag": "0014_add_product_type_to_agents",
+      "hash": "9102dcc2b9671a78a1691ab6d35e92c1fc560032d2df86103ed597521db20419"
+    },
+    {
+      "tag": "0015_new_v1_tables",
+      "hash": "2c6a834de43d4d87c3e857440ede60a725f9f314a0521b9b5ff75e9b152dc04d"
+    },
+    {
+      "tag": "0016_document_types",
+      "hash": "f96cf97918fc694318b8b6f94d0c3d9ce55daa7e01048ec32cb9755bad786e76"
+    },
+    {
+      "tag": "0017_document_metadata",
+      "hash": "7da47750df213b13956bdcbb7573196b81d84bc6bdd0da3991d3510f2f9090af"
+    },
+    {
+      "tag": "0018_identity_graph",
+      "hash": "f32c53f18a9d9e5ca41cc7aed3960d6c114bb468f0abc3de5c69fa40ddc3811d"
+    },
+    {
+      "tag": "0019_db_driven_prompts",
+      "hash": "fc0ff0e6057e5142b0a4ef5efb39737d6757ad7cd4a744d594e655b3e61e5708"
+    },
+    {
+      "tag": "0020_pending_extractions",
+      "hash": "c811cb83038597a8d533bc759338fcd7e8515e3a0312e1860d9ab898f7f9ccdb"
+    },
+    {
+      "tag": "0021_rag_documents",
+      "hash": "0c72555507227726e03003536f059ceeb833644d1f10d8474b3599ab9a8b9636"
+    },
+    {
+      "tag": "0022_user_quotas_usage_jsonb",
+      "hash": "73664463ec2d4c7a998a8adb44925367bff4d9f6c46b49e28903b1960edd2084"
+    },
+    {
+      "tag": "0023_user_facts_user_scoped_unique",
+      "hash": "6f8f6a7e823a298e7a6fa9bc9a124abc557516bd442a637d91801eb87233c7ec"
+    },
+    {
+      "tag": "0024_user_memories_hnsw",
+      "hash": "cffca18b0e4f2081546ae8cd19d027b0fcb5372dc16e6536f2c759ad6e1d391e"
+    },
+    {
+      "tag": "0025_audit_trigger_errcode",
+      "hash": "eb45e223eabce2f692587967ce24ea31c152bd1ecf50dfe60b28c7a8fc6d3443"
+    },
+    {
+      "tag": "0026_user_profile_snapshots_composite_pk",
+      "hash": "09dc229039b2e48df073c9cd181e58aa2c210357e077413aecfa70dc482687ee"
+    },
+    {
+      "tag": "0027_billing_settings_mnt_columns",
+      "hash": "f21baf51c21bf23e2ddd13985f06cf1acefd2f5fb6bf4fbc4c925d49f62051ae"
+    },
+    {
+      "tag": "0028_trial_credits_normalized_email",
+      "hash": "17c557228ee81f27073a537b5b534abf4557a27e7755a367cba566b14c307235"
+    },
+    {
+      "tag": "0029_usage_records_conversation_idx",
+      "hash": "260c39df837b8f21feddd35ef2263f52bc3821f288466b15e80d95f20bfe1e72"
+    },
+    {
+      "tag": "0030_pending_extractions_composite_idx",
+      "hash": "d0e398772a42f6e5de6b3f2a5562b9d98c996e72489413f51a2bb244f4317dda"
+    },
+    {
+      "tag": "0031_rag_documents_hnsw",
+      "hash": "bc96dba2f49403281d1f27e6f8ac1f55f22d34a3e701af19be54e666f4b088b1"
+    },
+    {
+      "tag": "0032_rate_limit_unique_idx",
+      "hash": "c2995c76b96b69d8dc6e8a676fe7c45c2fc66d7ed17ce8a68914f8c756869399"
+    },
+    {
+      "tag": "0033_pending_extractions_partial_idx",
+      "hash": "b74373ba8076cc1f38c3805063023e8dc89de11f3d98502f818460354de18f39"
+    },
+    {
+      "tag": "0034_rate_limit_count",
+      "hash": "6263b9451b91500213a5559ad960f9186b2fee9d73eeb30a4b3209ae1d3366cc"
+    },
+    {
+      "tag": "0035_credit_reservations",
+      "hash": "fe95a71462bde8980924e77aff20e38cbee92b4bbae210c504c83015720b10ff"
+    },
+    {
+      "tag": "0036_executions_audit_jobs",
+      "hash": "8665606757ec96bbd48eaab2b2ebb1ee5b5a86d58ba73635b3229ff94c40e572",
+      "alternates": [
+        "c93a93db58737ef13f25138b107711a86f4f21efb48d1c83c18f2ffa07831c3d"
+      ]
+    },
+    {
+      "tag": "0037_rate_limit_minute_bucket",
+      "hash": "112683e434f07c4db20179ebaffdc52d6f21f644e0573b718b2ff060794db064"
+    },
+    {
+      "tag": "0038_user_quotas_generic_usage",
+      "hash": "658b6f64424522d955f2aa4d2ce1fe32b52f0e5adbdba14af75c1678389d7754",
+      "alternates": [
+        "822df887955ddfdb138146ae47612dd0c37c5419b67223b6309b82253efc1c53"
+      ]
+    },
+    {
+      "tag": "0039_platform_admin_impersonation",
+      "hash": "e9d67dae6b4e0c1149aadd9f29d23a48c93861cfd5d176d61ca3ade00d735617"
+    },
+    {
+      "tag": "0040_ee_audit_chain",
+      "hash": "16b01fa704f75e13fe85cee307a2dd287e3cbd93e77fd58ce0eb18cc32e178e6",
+      "alternates": [
+        "3a432628f1dcf4c4efbda0421901421fdaaa99c498abdb22ee21c6baef24a005"
+      ]
+    },
+    {
+      "tag": "0041_audit_events_append_only",
+      "hash": "963fc9a3c589713ad74e8eed1000e5259f920bba908720d7628f56ad2fb81c40"
+    },
+    {
+      "tag": "0042_sessions_active_organization_id",
+      "hash": "31b737d2d7ba48959f7daeff2a4a162ed141cb8bfc933a0e1dfd49528295849c"
+    },
+    {
+      "tag": "0043_attachments",
+      "hash": "922c9c29097b128bfbdf012398541440304f06cfcc9a4ce3a8a9d7adc9f8fab2"
+    },
+    {
+      "tag": "0044_money_micros",
+      "hash": "794d552ba21338553376a7d0cc926d07bfe493b51afad48bd647b962d38efe51"
+    },
+    {
+      "tag": "0045_drop_legacy_money_columns",
+      "hash": "f50cf0e58acc2993693ece80bb6ccf21f738f8c828aed3dcad7e88bc30bef40c"
+    },
+    {
+      "tag": "0046_user_quotas_per_workspace",
+      "hash": "96ddd49d5163edb23328b703fe15d7397a319e43aa104ae4184d1221096b5a63"
+    }
+  ]
+}