@rebasepro/server-postgres 0.17.2-canary.g439a156 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{backup-service-BtgHxfFm.js → backup-service-DCk7KhhL.js} +3 -3
- package/dist/{backup-service-BtgHxfFm.js.map → backup-service-DCk7KhhL.js.map} +1 -1
- package/dist/cli-helpers.d.ts +26 -14
- package/dist/{collection-index-DxJBvVTH.js → collection-index-BRUg10H5.js} +2 -2
- package/dist/{collection-index-DxJBvVTH.js.map → collection-index-BRUg10H5.js.map} +1 -1
- package/dist/{ensure-collection-policies-BHk4TRuf.js → ensure-collection-policies-UCqgv_8c.js} +4 -4
- package/dist/{ensure-collection-policies-BHk4TRuf.js.map → ensure-collection-policies-UCqgv_8c.js.map} +1 -1
- package/dist/{ensure-collection-tables-DMjOkeRy.js → ensure-collection-tables-DgVixhX3.js} +256 -38
- package/dist/ensure-collection-tables-DgVixhX3.js.map +1 -0
- package/dist/index.es.js +12 -12
- package/dist/{rls-bootstrap-sql-DNzaWd4C.js → rls-bootstrap-sql-B5C9LoJ6.js} +2 -2
- package/dist/{rls-bootstrap-sql-DNzaWd4C.js.map → rls-bootstrap-sql-B5C9LoJ6.js.map} +1 -1
- package/dist/{rls-enforcement-C1RJ1uI2.js → rls-enforcement-DvAbL9YJ.js} +3 -3
- package/dist/{rls-enforcement-C1RJ1uI2.js.map → rls-enforcement-DvAbL9YJ.js.map} +1 -1
- package/dist/schema/atlas-argv.d.ts +58 -0
- package/dist/schema/carved-out-migration.d.ts +65 -0
- package/dist/schema/ensure-collection-tables.d.ts +13 -1
- package/dist/schema/generate-postgres-ddl-logic.d.ts +57 -5
- package/dist/schema/vector-index.d.ts +120 -0
- package/dist/{src-DiDgtX8P.js → src-DiB5RP2Z.js} +33 -3
- package/dist/{src-DiDgtX8P.js.map → src-DiB5RP2Z.js.map} +1 -1
- package/dist/{websocket-g1Ji7m4o.js → websocket-BZ4H5wUz.js} +19 -9
- package/dist/websocket-BZ4H5wUz.js.map +1 -0
- package/package.json +6 -6
- package/src/cli-helpers.ts +65 -34
- package/src/cli.ts +168 -71
- package/src/schema/atlas-argv.ts +94 -0
- package/src/schema/carved-out-migration.ts +404 -0
- package/src/schema/ensure-collection-tables.ts +59 -32
- package/src/schema/generate-postgres-ddl-logic.ts +147 -21
- package/src/schema/generate-postgres-ddl.ts +59 -6
- package/src/schema/generate-schema-commit.ts +31 -6
- package/src/schema/vector-index.ts +213 -0
- package/dist/ensure-collection-tables-DMjOkeRy.js.map +0 -1
- package/dist/websocket-g1Ji7m4o.js.map +0 -1
|
@@ -16,8 +16,15 @@ import {
|
|
|
16
16
|
type SearchColumnSpec
|
|
17
17
|
} from "./search-column";
|
|
18
18
|
import {
|
|
19
|
+
buildVectorColumnSpecs,
|
|
19
20
|
buildVectorIndexPlan,
|
|
21
|
+
vectorColumnDefinition,
|
|
22
|
+
vectorDimensionGuard,
|
|
23
|
+
vectorExtensionDeclared,
|
|
24
|
+
vectorExtensionStatement,
|
|
20
25
|
vectorIndexStatements,
|
|
26
|
+
vectorObjectNames,
|
|
27
|
+
VECTOR_EXTENSION_OPT_IN,
|
|
21
28
|
type VectorIndexPlan
|
|
22
29
|
} from "./vector-index";
|
|
23
30
|
import { buildCollectionIndexSpecs, collectionIndexStatements } from "./collection-index";
|
|
@@ -297,22 +304,15 @@ export const getSqlColumnType = (propName: string, prop: Property, collection: C
|
|
|
297
304
|
* appended to migrations that run against databases at any stage of their
|
|
298
305
|
* life. Empty when nothing opted in — the caller writes no file then.
|
|
299
306
|
*
|
|
300
|
-
* The
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
307
|
+
* The dev database Atlas plans against needs none of this. `searchExcludePatterns`
|
|
308
|
+
* removes the search column from the *inspected* state, so the state Atlas
|
|
309
|
+
* replays there never mentions the column and never calls a helper. Seeding the
|
|
310
|
+
* helpers into that database was tried and removed: measured on Atlas 1.2.3, a
|
|
311
|
+
* `schema apply --dry-run` produces byte-identical output seeded or not, and
|
|
312
|
+
* Atlas wipes the dev database before it plans anyway — so a seed could not have
|
|
313
|
+
* helped even where the excludes are missing, which is the case that really does
|
|
314
|
+
* fail with `function public.rebase_search_unaccent(text) does not exist`.
|
|
304
315
|
*/
|
|
305
|
-
export const searchPrerequisiteStatements = (allCollections: CollectionConfig[]): string[] => {
|
|
306
|
-
const specs = relationalCollections(allCollections)
|
|
307
|
-
.map(c => buildSearchColumnSpec(c))
|
|
308
|
-
.filter((s): s is SearchColumnSpec => s !== undefined);
|
|
309
|
-
if (specs.length === 0) return [];
|
|
310
|
-
return [
|
|
311
|
-
...new Set(specs.flatMap(searchExtensionStatements)),
|
|
312
|
-
...new Set(specs.flatMap(searchHelperFunctions))
|
|
313
|
-
];
|
|
314
|
-
};
|
|
315
|
-
|
|
316
316
|
export const generatePostgresSearchDdl = (allCollections: CollectionConfig[]): string => {
|
|
317
317
|
const collections = relationalCollections(allCollections);
|
|
318
318
|
const specs = collections
|
|
@@ -391,11 +391,112 @@ export const searchExcludePatterns = (allCollections: CollectionConfig[]): strin
|
|
|
391
391
|
return patterns;
|
|
392
392
|
};
|
|
393
393
|
|
|
394
|
+
/**
|
|
395
|
+
* Everything a `{ type: "vector" }` property needs, as a file Rebase applies
|
|
396
|
+
* itself.
|
|
397
|
+
*
|
|
398
|
+
* Vector is the second part of the schema Atlas does not own, and it is out for
|
|
399
|
+
* a narrower reason than search: not that Atlas refuses to parse the file, but
|
|
400
|
+
* that it cannot *execute* it. To compute a desired state Atlas materialises
|
|
401
|
+
* `schema.sql` in a dev database, that database is created empty, and Atlas
|
|
402
|
+
* empties it again at the start of every run — so `VECTOR(384)` resolves
|
|
403
|
+
* against a database that structurally cannot have pgvector, and every push
|
|
404
|
+
* dies with `type "vector" does not exist`. Seeding the extension beforehand
|
|
405
|
+
* does not survive the clean (measured: present before the run, gone after),
|
|
406
|
+
* putting `CREATE EXTENSION` in `schema.sql` is refused as an Atlas Pro
|
|
407
|
+
* feature, and `--exclude` alone cannot help because Atlas still parses and
|
|
408
|
+
* applies the whole file.
|
|
409
|
+
*
|
|
410
|
+
* So the column, its ANN indexes and the extension are excluded from Atlas's
|
|
411
|
+
* view ({@link vectorExcludePatterns}) and applied from here — the same
|
|
412
|
+
* arrangement search and the RLS policies already use.
|
|
413
|
+
*
|
|
414
|
+
* `CREATE EXTENSION` appears only when the project's database named `vector` in
|
|
415
|
+
* its `extensions` — see `DatabaseOptions.extensions`. Everything else is
|
|
416
|
+
* emitted either way: a database that already has pgvector installed by hand
|
|
417
|
+
* needs no permission from anyone, and one that does not gets the column
|
|
418
|
+
* refused by Postgres with a message this repo makes name the opt-in.
|
|
419
|
+
*
|
|
420
|
+
* Ordered as it must run: the extension, then the drift guard, then the column
|
|
421
|
+
* whose type needs the extension, then the indexes over that column. Every
|
|
422
|
+
* statement is `IF NOT EXISTS`, because this is replayed on every push and
|
|
423
|
+
* appended to migrations that run against databases at any stage of their life.
|
|
424
|
+
* Empty when no collection declares a vector — the caller writes no file then.
|
|
425
|
+
*/
|
|
426
|
+
export const generatePostgresVectorDdl = (
|
|
427
|
+
allCollections: CollectionConfig[],
|
|
428
|
+
options: { extensions?: readonly string[] } = {}
|
|
429
|
+
): string => {
|
|
430
|
+
const collections = relationalCollections(allCollections);
|
|
431
|
+
const withVectors = collections.filter(c => buildVectorColumnSpecs(c, resolveColumnName).length > 0);
|
|
432
|
+
if (withVectors.length === 0) return "";
|
|
433
|
+
|
|
434
|
+
let ddl = "-- This file is auto-generated by the Rebase DDL generator. Do not edit manually.\n";
|
|
435
|
+
ddl += "--\n";
|
|
436
|
+
ddl += "-- Vector columns and their ANN indexes, for the collections declaring a\n";
|
|
437
|
+
ddl += "-- `vector` property. Applied by Rebase, not by Atlas — see\n";
|
|
438
|
+
ddl += "-- generatePostgresVectorDdl.\n\n";
|
|
439
|
+
// The note is not decoration. Someone reading this file after a failed push
|
|
440
|
+
// is looking for the reason the type does not exist, and "no CREATE
|
|
441
|
+
// EXTENSION here" is invisible unless it is written down.
|
|
442
|
+
ddl += vectorExtensionDeclared(options.extensions)
|
|
443
|
+
? `${vectorExtensionStatement()}\n\n`
|
|
444
|
+
: `-- pgvector is not installed from here: no database declared it. To let Rebase\n` +
|
|
445
|
+
`-- install it, add ${VECTOR_EXTENSION_OPT_IN} in config/resources.ts.\n` +
|
|
446
|
+
`-- Otherwise install it once by hand; the column below needs the type either way.\n\n`;
|
|
447
|
+
|
|
448
|
+
for (const collection of withVectors) {
|
|
449
|
+
const table = `"${isPostgresCollectionConfig(collection) && collection.schema ? collection.schema : "public"}"."${getTableName(collection)}"`;
|
|
450
|
+
for (const spec of buildVectorColumnSpecs(collection, resolveColumnName)) {
|
|
451
|
+
// Guard before the ADD, so a changed `dimensions` is reported rather
|
|
452
|
+
// than skipped over — see `vectorDimensionGuard`.
|
|
453
|
+
ddl += `${vectorDimensionGuard(spec)}\n`;
|
|
454
|
+
// ADD COLUMN IF NOT EXISTS rather than the inline definition the
|
|
455
|
+
// CREATE TABLE would use: by the time this runs the table exists,
|
|
456
|
+
// whether Atlas just created it or it has been live for a year.
|
|
457
|
+
ddl += `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${vectorColumnDefinition(spec)};\n`;
|
|
458
|
+
}
|
|
459
|
+
const plan = buildVectorIndexPlan(collection, resolveColumnName);
|
|
460
|
+
vectorIndexStatements(plan).forEach(s => { ddl += `${s}\n`; });
|
|
461
|
+
for (const skip of plan.skipped) {
|
|
462
|
+
ddl += `-- No ANN index on "${skip.schema}"."${skip.table}"."${skip.column}": ${skip.reason}\n`;
|
|
463
|
+
}
|
|
464
|
+
ddl += "\n";
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
return ddl;
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Glob patterns telling Atlas to leave the vector column and its ANN indexes
|
|
472
|
+
* alone.
|
|
473
|
+
*
|
|
474
|
+
* Without these, a desired state that omits the column reads to Atlas as an
|
|
475
|
+
* instruction to drop it, taking the embeddings and the indexes with it on the
|
|
476
|
+
* next push.
|
|
477
|
+
*
|
|
478
|
+
* Fully qualified, `schema.table.object`, for the reason spelled out on
|
|
479
|
+
* {@link searchExcludePatterns}: the two-part form matches nothing and reports
|
|
480
|
+
* no error, which turns a protection into a `DROP COLUMN`.
|
|
481
|
+
*/
|
|
482
|
+
export const vectorExcludePatterns = (allCollections: CollectionConfig[]): string[] => {
|
|
483
|
+
const patterns: string[] = [];
|
|
484
|
+
for (const collection of relationalCollections(allCollections)) {
|
|
485
|
+
const schema = isPostgresCollectionConfig(collection) && collection.schema ? collection.schema : "public";
|
|
486
|
+
const table = getTableName(collection);
|
|
487
|
+
for (const name of vectorObjectNames(collection, resolveColumnName)) {
|
|
488
|
+
patterns.push(`${schema}.${table}.${name}`);
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
return patterns;
|
|
492
|
+
};
|
|
493
|
+
|
|
394
494
|
export const generatePostgresDdl = async (
|
|
395
495
|
allCollections: CollectionConfig[],
|
|
396
|
-
options: { includePolicies?: boolean; includeSearch?: boolean } = {
|
|
496
|
+
options: { includePolicies?: boolean; includeSearch?: boolean; includeVector?: boolean } = {
|
|
397
497
|
includePolicies: true,
|
|
398
|
-
includeSearch: true
|
|
498
|
+
includeSearch: true,
|
|
499
|
+
includeVector: true
|
|
399
500
|
}
|
|
400
501
|
): Promise<string> => {
|
|
401
502
|
// Only the collections this engine stores. See `relationalCollections`.
|
|
@@ -446,6 +547,18 @@ export const generatePostgresDdl = async (
|
|
|
446
547
|
}
|
|
447
548
|
}
|
|
448
549
|
|
|
550
|
+
// 1c. Vector support, for collections that declared a `vector` property.
|
|
551
|
+
//
|
|
552
|
+
// Only a note, never a statement — unlike 1b, which emits search's
|
|
553
|
+
// extensions when they are included. `CREATE EXTENSION vector` is the one
|
|
554
|
+
// thing this file must never carry whichever way the flag goes: Atlas
|
|
555
|
+
// rejects an extension in a desired state as a paid feature. The note is
|
|
556
|
+
// for whoever opens the generated file looking for the column.
|
|
557
|
+
if (options.includeVector === false
|
|
558
|
+
&& collections.some(c => buildVectorColumnSpecs(c, resolveColumnName).length > 0)) {
|
|
559
|
+
ddl += "-- Vector columns and their ANN indexes live in `vector.sql`, applied separately.\n\n";
|
|
560
|
+
}
|
|
561
|
+
|
|
449
562
|
// 2. Generate Enums
|
|
450
563
|
//
|
|
451
564
|
// The enum type name is derived from table + column, so two collections
|
|
@@ -579,6 +692,13 @@ export const generatePostgresDdl = async (
|
|
|
579
692
|
const columns: string[] = [];
|
|
580
693
|
|
|
581
694
|
Object.entries(collection.properties ?? {}).forEach(([propName, prop]) => {
|
|
695
|
+
// A vector column is not Atlas's — `vector.sql` adds it. Skipped
|
|
696
|
+
// here rather than filtered upstream so the column keeps its
|
|
697
|
+
// place in the generator's order when it *is* included, which is
|
|
698
|
+
// what the derived-names contract renders.
|
|
699
|
+
if (prop.type === "vector" && options.includeVector === false) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
582
702
|
if (prop.type === "relation") {
|
|
583
703
|
const refProp = prop as RelationProperty;
|
|
584
704
|
const resolvedRelations = resolveCollectionRelations(collection);
|
|
@@ -729,10 +849,16 @@ export const generatePostgresDdl = async (
|
|
|
729
849
|
// rather than inline, because `CREATE INDEX` is a statement and a
|
|
730
850
|
// column definition is not — and because a column too wide for
|
|
731
851
|
// pgvector to index still needs its column.
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
852
|
+
//
|
|
853
|
+
// Absent alongside their column when vector is carved out: an index
|
|
854
|
+
// over a column this file no longer declares would not resolve, and
|
|
855
|
+
// `USING hnsw` needs the extension the dev database cannot have.
|
|
856
|
+
if (options.includeVector !== false) {
|
|
857
|
+
const vectorPlan: VectorIndexPlan = buildVectorIndexPlan(collection, resolveColumnName);
|
|
858
|
+
indexStatements.push(...vectorIndexStatements(vectorPlan));
|
|
859
|
+
for (const skip of vectorPlan.skipped) {
|
|
860
|
+
indexStatements.push(`-- No ANN index on "${skip.schema}"."${skip.table}"."${skip.column}": ${skip.reason}`);
|
|
861
|
+
}
|
|
736
862
|
}
|
|
737
863
|
|
|
738
864
|
// The collection's own `indexes:` block. Last of the three index
|
|
@@ -5,12 +5,43 @@ import { pathToFileURL } from "url";
|
|
|
5
5
|
import {
|
|
6
6
|
generatePostgresDdl,
|
|
7
7
|
generatePostgresPoliciesDdl,
|
|
8
|
-
generatePostgresSearchDdl
|
|
8
|
+
generatePostgresSearchDdl,
|
|
9
|
+
generatePostgresVectorDdl
|
|
9
10
|
} from "./generate-postgres-ddl-logic";
|
|
10
|
-
import { CollectionConfig } from "@rebasepro/types";
|
|
11
|
+
import { CollectionConfig, declaredDatabaseExtensions } from "@rebasepro/types";
|
|
11
12
|
import { loadCollectionsFromDirectory } from "@rebasepro/server";
|
|
12
13
|
import { out, outError } from "../cli-output";
|
|
13
14
|
|
|
15
|
+
/** Where a project declares its resources, relative to the config directory. */
|
|
16
|
+
const RESOURCE_ENTRIES = ["resources.ts", "resources.js", "resources/index.ts", "resources/index.js"];
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Evaluate the project's `resources.ts`, so `declaredDatabaseExtensions()` can
|
|
20
|
+
* answer.
|
|
21
|
+
*
|
|
22
|
+
* This generator runs as its own tsx subprocess and loads only the collections
|
|
23
|
+
* directory, so nothing else in it would ever import the resources module — and
|
|
24
|
+
* a permission that reads as absent because nobody loaded it is the worst shape
|
|
25
|
+
* this could take: the push succeeds, `vector.sql` quietly omits the
|
|
26
|
+
* `CREATE EXTENSION`, and the failure lands one statement later looking like a
|
|
27
|
+
* database problem.
|
|
28
|
+
*
|
|
29
|
+
* Imported directly rather than through the config index, for the reason
|
|
30
|
+
* `boot/resource-loading.ts` gives: re-exporting from the index would work only
|
|
31
|
+
* for projects that remember to.
|
|
32
|
+
*
|
|
33
|
+
* A project with no resources module is the normal case and declares nothing.
|
|
34
|
+
* An import that *throws* is not swallowed — a resources module that does not
|
|
35
|
+
* evaluate is a configuration error, and treating it as "declared nothing"
|
|
36
|
+
* would turn it into a silently missing extension.
|
|
37
|
+
*/
|
|
38
|
+
async function loadDeclaredResources(configDir: string): Promise<void> {
|
|
39
|
+
const entry = RESOURCE_ENTRIES
|
|
40
|
+
.map(name => path.join(configDir, name))
|
|
41
|
+
.find(candidate => fs.existsSync(candidate));
|
|
42
|
+
if (!entry) return;
|
|
43
|
+
await import(pathToFileURL(entry).href);
|
|
44
|
+
}
|
|
14
45
|
|
|
15
46
|
const runGeneration = async (collectionsFilePath?: string, outputPath?: string) => {
|
|
16
47
|
try {
|
|
@@ -21,6 +52,11 @@ const runGeneration = async (collectionsFilePath?: string, outputPath?: string)
|
|
|
21
52
|
|
|
22
53
|
const resolvedPath = path.resolve(collectionsFilePath);
|
|
23
54
|
|
|
55
|
+
// `<config>/collections` is the path this is given, so the config
|
|
56
|
+
// directory is its parent — the same relationship `rebase resources`
|
|
57
|
+
// and the boot loader assume.
|
|
58
|
+
await loadDeclaredResources(path.dirname(resolvedPath));
|
|
59
|
+
|
|
24
60
|
// Shared with the runtime and the doctor: what gets generated here must
|
|
25
61
|
// be exactly what the server serves, including directory-level defaults.
|
|
26
62
|
let collections: CollectionConfig[] = await loadCollectionsFromDirectory(resolvedPath);
|
|
@@ -34,15 +70,19 @@ const runGeneration = async (collectionsFilePath?: string, outputPath?: string)
|
|
|
34
70
|
collections.sort((a, b) => a.slug.localeCompare(b.slug));
|
|
35
71
|
|
|
36
72
|
// `schema.sql` is Atlas's desired state, and it carries neither the RLS
|
|
37
|
-
// policies nor the search apparatus
|
|
38
|
-
//
|
|
39
|
-
// in their own right.
|
|
73
|
+
// policies, nor the search apparatus, nor the vector columns: Atlas
|
|
74
|
+
// manages none of them, and in the search and vector cases cannot. All
|
|
75
|
+
// three are written beside it and applied by the CLI in their own right.
|
|
40
76
|
const ddlContent = await generatePostgresDdl(collections, {
|
|
41
77
|
includePolicies: false,
|
|
42
|
-
includeSearch: false
|
|
78
|
+
includeSearch: false,
|
|
79
|
+
includeVector: false
|
|
43
80
|
});
|
|
44
81
|
const policiesContent = generatePostgresPoliciesDdl(collections);
|
|
45
82
|
const searchContent = generatePostgresSearchDdl(collections);
|
|
83
|
+
const vectorContent = generatePostgresVectorDdl(collections, {
|
|
84
|
+
extensions: declaredDatabaseExtensions()
|
|
85
|
+
});
|
|
46
86
|
|
|
47
87
|
if (outputPath) {
|
|
48
88
|
const outputDir = path.dirname(outputPath);
|
|
@@ -64,6 +104,19 @@ const runGeneration = async (collectionsFilePath?: string, outputPath?: string)
|
|
|
64
104
|
} else if (fs.existsSync(searchPath)) {
|
|
65
105
|
await fsPromises.rm(searchPath);
|
|
66
106
|
}
|
|
107
|
+
|
|
108
|
+
// Removed when the last `vector` property goes, for the same reason
|
|
109
|
+
// `search.sql` is: the CLI applies whichever of these files exists,
|
|
110
|
+
// and a stale one would keep re-adding a column the collections no
|
|
111
|
+
// longer declare — which Atlas, no longer told to exclude it, would
|
|
112
|
+
// then plan a DROP for on the very next push.
|
|
113
|
+
const vectorPath = path.join(outputDir, "vector.sql");
|
|
114
|
+
if (vectorContent) {
|
|
115
|
+
await fsPromises.writeFile(vectorPath, vectorContent);
|
|
116
|
+
out(`✅ PostgreSQL vector DDL generated successfully at ${vectorPath}`);
|
|
117
|
+
} else if (fs.existsSync(vectorPath)) {
|
|
118
|
+
await fsPromises.rm(vectorPath);
|
|
119
|
+
}
|
|
67
120
|
} else {
|
|
68
121
|
out("✅ PostgreSQL DDL generated successfully.");
|
|
69
122
|
out(String(ddlContent));
|
|
@@ -37,11 +37,17 @@
|
|
|
37
37
|
* migration this module cannot make valid would be worse than handing the
|
|
38
38
|
* statements to a caller who knows which kind of project it is.
|
|
39
39
|
*/
|
|
40
|
-
import {
|
|
40
|
+
import {
|
|
41
|
+
DEFAULT_COMMIT_PATHS,
|
|
42
|
+
declaredDatabaseExtensions,
|
|
43
|
+
type CollectionConfig,
|
|
44
|
+
type SchemaCommitPaths
|
|
45
|
+
} from "@rebasepro/types";
|
|
41
46
|
import {
|
|
42
47
|
generatePostgresDdl,
|
|
43
48
|
generatePostgresPoliciesDdl,
|
|
44
|
-
generatePostgresSearchDdl
|
|
49
|
+
generatePostgresSearchDdl,
|
|
50
|
+
generatePostgresVectorDdl
|
|
45
51
|
} from "./generate-postgres-ddl-logic";
|
|
46
52
|
import { generateSchema } from "./generate-drizzle-schema-logic";
|
|
47
53
|
import {
|
|
@@ -207,18 +213,37 @@ export async function generateSchemaCommit(input: SchemaCommitInput): Promise<Sc
|
|
|
207
213
|
);
|
|
208
214
|
}
|
|
209
215
|
|
|
210
|
-
|
|
216
|
+
// The same split `rebase db generate` writes, and it has to be the same:
|
|
217
|
+
// these files land in the repository, and the next `db push` runs Atlas
|
|
218
|
+
// against `schema.sql`. Generated whole, it carries the RLS policies (which
|
|
219
|
+
// Atlas would then own and drop), the search helpers (which its free tier
|
|
220
|
+
// refuses to parse at all) and `VECTOR(n)` (which its dev database cannot
|
|
221
|
+
// resolve) — so a live schema edit used to leave behind a desired-state
|
|
222
|
+
// file that `db push` chokes on.
|
|
223
|
+
const [schema, ddl, policies, search, vector] = await Promise.all([
|
|
211
224
|
generateSchema(input.after),
|
|
212
|
-
generatePostgresDdl(input.after
|
|
225
|
+
generatePostgresDdl(input.after, {
|
|
226
|
+
includePolicies: false,
|
|
227
|
+
includeSearch: false,
|
|
228
|
+
includeVector: false
|
|
229
|
+
}),
|
|
213
230
|
Promise.resolve(generatePostgresPoliciesDdl(input.after)),
|
|
214
|
-
Promise.resolve(generatePostgresSearchDdl(input.after))
|
|
231
|
+
Promise.resolve(generatePostgresSearchDdl(input.after)),
|
|
232
|
+
// The registry, because this runs inside the booted server, which has
|
|
233
|
+
// already evaluated the project's `resources.ts`. A commit that wrote a
|
|
234
|
+
// `vector.sql` disagreeing with the one `rebase db generate` produces
|
|
235
|
+
// would show up as drift in the repository the moment anyone regenerated.
|
|
236
|
+
Promise.resolve(generatePostgresVectorDdl(input.after, {
|
|
237
|
+
extensions: declaredDatabaseExtensions()
|
|
238
|
+
}))
|
|
215
239
|
]);
|
|
216
240
|
|
|
217
241
|
const generated: SchemaCommitFile[] = [
|
|
218
242
|
{ path: paths.schemaFile, contents: schema },
|
|
219
243
|
{ path: paths.ddlFile, contents: ddl },
|
|
220
244
|
{ path: paths.policiesFile, contents: policies },
|
|
221
|
-
{ path: paths.searchFile, contents: search }
|
|
245
|
+
{ path: paths.searchFile, contents: search },
|
|
246
|
+
{ path: paths.vectorFile, contents: vector }
|
|
222
247
|
];
|
|
223
248
|
|
|
224
249
|
// Both sides planned once, here, rather than through `additiveStatements` —
|
|
@@ -28,6 +28,17 @@
|
|
|
28
28
|
* boot-time ensure render the *same* specification rather than describing the
|
|
29
29
|
* same index twice, differently. `contracts/derived-names.txt` records the
|
|
30
30
|
* names both produce, and CI fails if they diverge.
|
|
31
|
+
*
|
|
32
|
+
* ## And why the column itself is described here too
|
|
33
|
+
*
|
|
34
|
+
* `VECTOR(n)` is a type Atlas cannot be shown. It has to materialise the
|
|
35
|
+
* desired state in a dev database to diff against, that database is created
|
|
36
|
+
* empty and *emptied again* by Atlas at the start of every run, and nothing in
|
|
37
|
+
* the free tier can put `CREATE EXTENSION vector` back — so a `schema.sql`
|
|
38
|
+
* mentioning the type fails with `type "vector" does not exist` on every push,
|
|
39
|
+
* for good. Search hit the same wall for its own reasons and took the same way
|
|
40
|
+
* out: the objects leave `schema.sql`, Atlas is told to exclude them, and
|
|
41
|
+
* Rebase applies them itself. See `generatePostgresVectorDdl`.
|
|
31
42
|
*/
|
|
32
43
|
import type { CollectionConfig, Property, VectorDistance, VectorIndexConfig } from "@rebasepro/types";
|
|
33
44
|
import { isPostgresCollectionConfig } from "@rebasepro/types";
|
|
@@ -276,3 +287,205 @@ export const vectorIndexStatements = (plan: VectorIndexPlan): string[] =>
|
|
|
276
287
|
/** The index names a plan creates — what the derived-names contract records. */
|
|
277
288
|
export const vectorIndexNames = (plan: VectorIndexPlan): string[] =>
|
|
278
289
|
plan.specs.map(spec => spec.indexName);
|
|
290
|
+
|
|
291
|
+
// ── The column, for the file Rebase applies itself ──────────────────────────
|
|
292
|
+
|
|
293
|
+
const quote = (v: string): string => `'${v.replace(/'/g, "''")}'`;
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* The schema pgvector's types are installed into.
|
|
297
|
+
*
|
|
298
|
+
* `WITH SCHEMA public` for the same load-bearing reason `searchExtensionStatements`
|
|
299
|
+
* gives: an unqualified `CREATE EXTENSION` lands in the first schema on
|
|
300
|
+
* `search_path`, which is `"$user", public` — and the scaffold's role is named
|
|
301
|
+
* `rebase`, the same as a schema the generator creates. Left to itself the
|
|
302
|
+
* extension would install into `rebase`, and every unqualified `VECTOR(n)`
|
|
303
|
+
* below would fail to resolve.
|
|
304
|
+
*/
|
|
305
|
+
export const VECTOR_EXTENSION_SCHEMA = "public";
|
|
306
|
+
|
|
307
|
+
/** The extension name a database has to name to let Rebase install pgvector. */
|
|
308
|
+
export const VECTOR_EXTENSION = "vector";
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* How a project says Rebase may install pgvector, quoted into the messages that
|
|
312
|
+
* have to name it. Spelled once so the option and the advice cannot drift.
|
|
313
|
+
*/
|
|
314
|
+
export const VECTOR_EXTENSION_OPT_IN = `database({ extensions: ["${VECTOR_EXTENSION}"] })`;
|
|
315
|
+
|
|
316
|
+
/** Did the project give Rebase leave to install pgvector? */
|
|
317
|
+
export const vectorExtensionDeclared = (extensions: readonly string[] | undefined): boolean =>
|
|
318
|
+
(extensions ?? []).includes(VECTOR_EXTENSION);
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Install pgvector — emitted only when the database asked for it.
|
|
322
|
+
*
|
|
323
|
+
* Opt-in because installing an extension is a decision with a deployment behind
|
|
324
|
+
* it: the image has to ship the library, the role has to be allowed to install
|
|
325
|
+
* it, and a managed provider has to have it on an allow-list. None of that is
|
|
326
|
+
* visible from inside the connection, so Rebase does not decide it. See
|
|
327
|
+
* `DatabaseOptions.extensions`.
|
|
328
|
+
*
|
|
329
|
+
* Withholding the statement is not withholding the *column*: the column is
|
|
330
|
+
* still created, and Postgres refuses it with `type "vector" does not exist`
|
|
331
|
+
* on a database where pgvector was never installed by hand. That error is the
|
|
332
|
+
* one this design accepts, and `vectorExtensionHint` is what makes it name
|
|
333
|
+
* {@link VECTOR_EXTENSION_OPT_IN} rather than nothing.
|
|
334
|
+
*/
|
|
335
|
+
export const vectorExtensionStatement = (): string =>
|
|
336
|
+
`CREATE EXTENSION IF NOT EXISTS ${VECTOR_EXTENSION} WITH SCHEMA ${VECTOR_EXTENSION_SCHEMA};`;
|
|
337
|
+
|
|
338
|
+
/**
|
|
339
|
+
* The missing-pgvector explanation, appended to whichever error revealed it.
|
|
340
|
+
*
|
|
341
|
+
* Two readers, needing opposite things, which is why the branch is on the error
|
|
342
|
+
* text rather than on the configuration:
|
|
343
|
+
*
|
|
344
|
+
* - **`type "vector" does not exist`** — nobody opted in and the database has
|
|
345
|
+
* no pgvector. The fix is one line of config they cannot guess, so naming
|
|
346
|
+
* the option *is* the hint.
|
|
347
|
+
* - **the install itself failed** — the config is already right, and repeating
|
|
348
|
+
* the option would send them to edit a correct line. What is missing is the
|
|
349
|
+
* library on the server, or the grant.
|
|
350
|
+
*
|
|
351
|
+
* Lives here rather than beside either caller because both need it: the boot
|
|
352
|
+
* ensure raises the first through its action applier, and `rebase db push`
|
|
353
|
+
* raises it out of `applyVectorDdl`. A hint on only one path is how a bare
|
|
354
|
+
* `type "vector" does not exist` reaches somebody — which is the thing this
|
|
355
|
+
* exists to prevent.
|
|
356
|
+
*/
|
|
357
|
+
export const vectorExtensionHint = (message: string): string => {
|
|
358
|
+
const installRefused = /extension "vector" is not available/i.test(message)
|
|
359
|
+
|| /(permission denied to create extension|must be (superuser|owner).{0,40}extension)/i.test(message);
|
|
360
|
+
if (installRefused) {
|
|
361
|
+
return (
|
|
362
|
+
"\n pgvector was declared and could not be installed. It is a server extension, so it needs an image " +
|
|
363
|
+
"that ships the library (the scaffold's `pgvector/pgvector:pg18` does; a stock `postgres:18` does not) " +
|
|
364
|
+
"and a role allowed to run `CREATE EXTENSION vector;`. Managed Postgres usually allows it once the " +
|
|
365
|
+
"extension is on the provider's allow-list."
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
if (!/type "(vector|halfvec|sparsevec)" does not exist/i.test(message)) return "";
|
|
369
|
+
return (
|
|
370
|
+
"\n pgvector is not installed on this database, and Rebase installs it only where a database says it may: " +
|
|
371
|
+
`add \`${VECTOR_EXTENSION_OPT_IN}\` in config/resources.ts, or install it once by hand with ` +
|
|
372
|
+
"`CREATE EXTENSION vector;`. Either way the server needs an image that ships the library — the scaffold's " +
|
|
373
|
+
"`pgvector/pgvector:pg18` does, a stock `postgres:18` does not. Rebase then creates the column and its ANN " +
|
|
374
|
+
"index automatically — see the `index` option on the property."
|
|
375
|
+
);
|
|
376
|
+
};
|
|
377
|
+
|
|
378
|
+
/** One `{ type: "vector" }` property, as a column. */
|
|
379
|
+
export interface VectorColumnSpec {
|
|
380
|
+
schema: string;
|
|
381
|
+
table: string;
|
|
382
|
+
column: string;
|
|
383
|
+
dimensions: number;
|
|
384
|
+
/** Rendered after the type, in the generator's order: UNIQUE then NOT NULL. */
|
|
385
|
+
modifiers: string;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Every vector column a collection declares.
|
|
390
|
+
*
|
|
391
|
+
* Wider than {@link buildVectorIndexPlan} on purpose: that one answers "what
|
|
392
|
+
* gets an ANN index", and skips both a property with `index: false` and one too
|
|
393
|
+
* wide to index. Either still needs its column.
|
|
394
|
+
*/
|
|
395
|
+
export const buildVectorColumnSpecs = (
|
|
396
|
+
collection: CollectionConfig,
|
|
397
|
+
resolveColumn: (propName: string, prop?: Property | null) => string
|
|
398
|
+
): VectorColumnSpec[] => {
|
|
399
|
+
const table = getTableName(collection);
|
|
400
|
+
const schema = isPostgresCollectionConfig(collection) && collection.schema ? collection.schema : "public";
|
|
401
|
+
const specs: VectorColumnSpec[] = [];
|
|
402
|
+
|
|
403
|
+
for (const [propName, prop] of Object.entries(collection.properties ?? {})) {
|
|
404
|
+
if (!isVectorProperty(prop)) continue;
|
|
405
|
+
// Assembled in the CREATE TABLE generator's order, so the column this
|
|
406
|
+
// file adds and the column `schema.sql` used to declare are the same
|
|
407
|
+
// column. A vector property is never a primary key and carries no
|
|
408
|
+
// SQL-level default, which is what leaves only these two.
|
|
409
|
+
const validation = (prop as { validation?: { required?: boolean; unique?: boolean } }).validation;
|
|
410
|
+
let modifiers = "";
|
|
411
|
+
if (validation?.unique) modifiers += " UNIQUE";
|
|
412
|
+
if (validation?.required) modifiers += " NOT NULL";
|
|
413
|
+
specs.push({
|
|
414
|
+
schema,
|
|
415
|
+
table,
|
|
416
|
+
column: resolveColumn(propName, prop as Property),
|
|
417
|
+
dimensions: prop.dimensions,
|
|
418
|
+
modifiers
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
return specs;
|
|
423
|
+
};
|
|
424
|
+
|
|
425
|
+
/** The column type — the one place `VECTOR(n)` is spelled. */
|
|
426
|
+
export const vectorColumnType = (spec: Pick<VectorColumnSpec, "dimensions">): string =>
|
|
427
|
+
`VECTOR(${spec.dimensions})`;
|
|
428
|
+
|
|
429
|
+
/** The column definition as it appears inside `CREATE TABLE`. */
|
|
430
|
+
export const vectorColumnDefinition = (spec: VectorColumnSpec): string =>
|
|
431
|
+
`"${spec.column}" ${vectorColumnType(spec)}${spec.modifiers}`;
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Refuse — or perform — a `dimensions` change that `ADD COLUMN IF NOT EXISTS`
|
|
435
|
+
* would otherwise swallow.
|
|
436
|
+
*
|
|
437
|
+
* Without this the file *launders* the change: the ADD is a no-op against a
|
|
438
|
+
* column that exists, so a project that went from 384 to 768 dimensions would
|
|
439
|
+
* push clean, keep a 384-wide column, and fail on the next insert with a
|
|
440
|
+
* message about the row rather than about the config. Atlas used to catch this
|
|
441
|
+
* — it owned the column and planned the `ALTER … TYPE` — and taking the column
|
|
442
|
+
* out of its sight is exactly what makes the guard necessary.
|
|
443
|
+
*
|
|
444
|
+
* The widening is performed when the column holds no values, because that is
|
|
445
|
+
* the case Atlas handled and it is the common one: a developer changing
|
|
446
|
+
* embedding models before there are any embeddings. With values present the
|
|
447
|
+
* conversion is pgvector's to reject — every stored vector is the old width —
|
|
448
|
+
* so this refuses first and names the statement to run afterwards.
|
|
449
|
+
*
|
|
450
|
+
* `atttypmod` on a `vector` column *is* the dimension count: pgvector stores it
|
|
451
|
+
* directly rather than offsetting it the way `varchar` does. `-1` means the
|
|
452
|
+
* column was declared as a bare `vector`, which is drift in the same sense.
|
|
453
|
+
*/
|
|
454
|
+
export const vectorDimensionGuard = (spec: VectorColumnSpec): string => {
|
|
455
|
+
const relation = `"${spec.schema}"."${spec.table}"`;
|
|
456
|
+
return `DO $rebase_vector$
|
|
457
|
+
DECLARE actual int;
|
|
458
|
+
BEGIN
|
|
459
|
+
SELECT a.atttypmod INTO actual
|
|
460
|
+
FROM pg_attribute a
|
|
461
|
+
JOIN pg_type t ON t.oid = a.atttypid
|
|
462
|
+
WHERE a.attrelid = ${quote(relation)}::regclass
|
|
463
|
+
AND a.attname = ${quote(spec.column)}
|
|
464
|
+
AND NOT a.attisdropped
|
|
465
|
+
AND t.typname = 'vector';
|
|
466
|
+
IF actual IS NOT NULL AND actual <> ${spec.dimensions} THEN
|
|
467
|
+
IF EXISTS (SELECT 1 FROM ${relation} WHERE "${spec.column}" IS NOT NULL) THEN
|
|
468
|
+
RAISE EXCEPTION 'Rebase: ${spec.schema}.${spec.table}."${spec.column}" declares ${spec.dimensions} dimensions but the column is vector(%), and it already holds values of the old width. pgvector cannot convert them. Re-embed the rows at ${spec.dimensions} dimensions (or clear the column), then re-apply: ALTER TABLE ${relation} ALTER COLUMN "${spec.column}" TYPE ${vectorColumnType(spec)};', actual;
|
|
469
|
+
END IF;
|
|
470
|
+
ALTER TABLE ${relation} ALTER COLUMN "${spec.column}" TYPE ${vectorColumnType(spec)};
|
|
471
|
+
END IF;
|
|
472
|
+
END
|
|
473
|
+
$rebase_vector$;`;
|
|
474
|
+
};
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* The object names a collection's vector properties own, unqualified.
|
|
478
|
+
*
|
|
479
|
+
* What Atlas has to be told to exclude. Only the column and the ANN indexes: a
|
|
480
|
+
* `UNIQUE` or `NOT NULL` on the column is a property *of* the column, and
|
|
481
|
+
* excluding the column takes them with it — measured against Atlas 1.2.3 and
|
|
482
|
+
* 1.3.2, where a target carrying `vector(3) NOT NULL UNIQUE` and a `schema.sql`
|
|
483
|
+
* carrying neither reported "Schema is synced, no changes to be made".
|
|
484
|
+
*/
|
|
485
|
+
export const vectorObjectNames = (
|
|
486
|
+
collection: CollectionConfig,
|
|
487
|
+
resolveColumn: (propName: string, prop?: Property | null) => string
|
|
488
|
+
): string[] => [
|
|
489
|
+
...buildVectorColumnSpecs(collection, resolveColumn).map(spec => spec.column),
|
|
490
|
+
...vectorIndexNames(buildVectorIndexPlan(collection, resolveColumn))
|
|
491
|
+
];
|