@pvidaal07/heydb 0.0.0-stage → 2.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/LICENSE +21 -0
- package/README.md +149 -2
- package/extensions/commands.ts +732 -0
- package/extensions/context.ts +37 -0
- package/extensions/deps.ts +32 -0
- package/extensions/heydb.ts +20 -0
- package/extensions/secret-input.ts +120 -0
- package/extensions/tools.ts +402 -0
- package/lib/about.ts +7 -0
- package/lib/config/operations.ts +75 -0
- package/lib/config/store.ts +73 -0
- package/lib/config/types.ts +37 -0
- package/lib/config/validate.ts +136 -0
- package/lib/engines/index.ts +66 -0
- package/lib/engines/mysql/catalog.ts +371 -0
- package/lib/engines/mysql/introspect.ts +186 -0
- package/lib/engines/postgres/catalog.ts +390 -0
- package/lib/engines/postgres/introspect.ts +240 -0
- package/lib/knowledge/operations.ts +223 -0
- package/lib/knowledge/query.ts +129 -0
- package/lib/knowledge/types.ts +64 -0
- package/lib/knowledge/validate.ts +151 -0
- package/lib/model/diff.ts +218 -0
- package/lib/model/hash.ts +88 -0
- package/lib/model/names.ts +141 -0
- package/lib/model/order.ts +47 -0
- package/lib/model/references.ts +251 -0
- package/lib/model/snapshot.ts +128 -0
- package/lib/model/stats.ts +97 -0
- package/lib/model/validate.ts +240 -0
- package/lib/render/markdown.ts +508 -0
- package/lib/search.ts +530 -0
- package/lib/secrets.ts +149 -0
- package/lib/store/envfile.ts +124 -0
- package/lib/store/files.ts +57 -0
- package/lib/store/guards.ts +68 -0
- package/lib/store/knowledge.ts +139 -0
- package/lib/store/project.ts +28 -0
- package/lib/store/snapshots.ts +142 -0
- package/lib/usecases/author.ts +51 -0
- package/lib/usecases/code-scan.ts +435 -0
- package/lib/usecases/init.ts +111 -0
- package/lib/usecases/knowledge.ts +214 -0
- package/lib/usecases/project.ts +377 -0
- package/lib/usecases/resolve.ts +186 -0
- package/lib/usecases/sharing.ts +83 -0
- package/lib/usecases/status.ts +37 -0
- package/lib/usecases/sync.ts +33 -0
- package/package.json +69 -5
- package/skills/heydb/SKILL.md +67 -0
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
import { formatName } from "../../model/names.ts";
|
|
2
|
+
import { sortSnapshot } from "../../model/order.ts";
|
|
3
|
+
import type {
|
|
4
|
+
Column,
|
|
5
|
+
ForeignKey,
|
|
6
|
+
Index,
|
|
7
|
+
ReferentialAction,
|
|
8
|
+
Routine,
|
|
9
|
+
Schema,
|
|
10
|
+
Snapshot,
|
|
11
|
+
Table,
|
|
12
|
+
Trigger,
|
|
13
|
+
Warning,
|
|
14
|
+
} from "../../model/snapshot.ts";
|
|
15
|
+
|
|
16
|
+
// Pure translation from PostgreSQL catalog rows to a Snapshot, like
|
|
17
|
+
// lib/engines/mysql/catalog.ts: introspect.ts runs the queries and hands the
|
|
18
|
+
// rows here, so every rule below is tested with hand-written rows.
|
|
19
|
+
//
|
|
20
|
+
// PostgreSQL has three levels, database → schema → table, and a connection
|
|
21
|
+
// only ever reaches the database it was opened on. So a PostgreSQL snapshot
|
|
22
|
+
// has exactly one Database (the connected one) with one Schema per captured
|
|
23
|
+
// schema, and every foreign key points inside that same database.
|
|
24
|
+
//
|
|
25
|
+
// Row field names match the aliases used in introspect.ts queries.
|
|
26
|
+
|
|
27
|
+
export interface PostgresCatalog {
|
|
28
|
+
/** Result of SHOW server_version, e.g. "16.4 (Debian 16.4-1.pgdg120+1)". */
|
|
29
|
+
readonly version: string;
|
|
30
|
+
readonly capturedAt: string;
|
|
31
|
+
/** The database the connection is on: the only one a snapshot can describe. */
|
|
32
|
+
readonly database: string;
|
|
33
|
+
/** Schemas the caller asked for; null means "every non-system schema". */
|
|
34
|
+
readonly requestedSchemas: readonly string[] | null;
|
|
35
|
+
/** Every schema of the database, system ones included. */
|
|
36
|
+
readonly schemas: readonly SchemaRow[];
|
|
37
|
+
readonly tables: readonly TableRow[];
|
|
38
|
+
readonly columns: readonly ColumnRow[];
|
|
39
|
+
readonly indexColumns: readonly IndexColumnRow[];
|
|
40
|
+
readonly foreignKeyColumns: readonly ForeignKeyColumnRow[];
|
|
41
|
+
readonly triggers: readonly TriggerRow[];
|
|
42
|
+
readonly routines: readonly RoutineRow[];
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface SchemaRow {
|
|
46
|
+
readonly schemaName: string;
|
|
47
|
+
/** has_schema_privilege(..., 'USAGE'): without it no table of the schema can be queried. */
|
|
48
|
+
readonly usable: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface TableRow {
|
|
52
|
+
readonly schemaName: string;
|
|
53
|
+
readonly tableName: string;
|
|
54
|
+
/** pg_class.relkind: r table, p partitioned table, f foreign table, v view, m materialized view. */
|
|
55
|
+
readonly relkind: string;
|
|
56
|
+
/** pg_class.reltuples: an estimate, -1 when the table was never analyzed (PostgreSQL 14+). */
|
|
57
|
+
readonly reltuples: number | string;
|
|
58
|
+
/** Sum of the known estimates of a partitioned table's partitions; null when none is known. */
|
|
59
|
+
readonly partitionRows: number | string | null;
|
|
60
|
+
readonly comment: string | null;
|
|
61
|
+
/** pg_get_viewdef for views and materialized views; null for anything else. */
|
|
62
|
+
readonly viewDefinition: string | null;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ColumnRow {
|
|
66
|
+
readonly schemaName: string;
|
|
67
|
+
readonly tableName: string;
|
|
68
|
+
readonly columnName: string;
|
|
69
|
+
readonly position: number;
|
|
70
|
+
/** format_type(): schema-qualified when the type is not in pg_catalog, e.g. "public.order_status". */
|
|
71
|
+
readonly dataType: string;
|
|
72
|
+
readonly notNull: boolean;
|
|
73
|
+
/** pg_get_expr of the default, or of the expression of a generated column. */
|
|
74
|
+
readonly defaultExpression: string | null;
|
|
75
|
+
/** pg_attribute.attidentity: "a" ALWAYS, "d" BY DEFAULT, "" not an identity column. */
|
|
76
|
+
readonly identity: string;
|
|
77
|
+
/** pg_attribute.attgenerated: "s" stored, "v" virtual (PostgreSQL 18), "" not generated. */
|
|
78
|
+
readonly generated: string;
|
|
79
|
+
readonly comment: string | null;
|
|
80
|
+
/** Only when it differs from the type's default collation, i.e. set explicitly. */
|
|
81
|
+
readonly collation: string | null;
|
|
82
|
+
/** Labels of an enum type in their declared order; null for any other type. */
|
|
83
|
+
readonly enumLabels: readonly string[] | null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface IndexColumnRow {
|
|
87
|
+
readonly schemaName: string;
|
|
88
|
+
readonly tableName: string;
|
|
89
|
+
readonly indexName: string;
|
|
90
|
+
readonly primary: boolean;
|
|
91
|
+
readonly unique: boolean;
|
|
92
|
+
/** Access method from pg_am: btree, hash, gin, gist… */
|
|
93
|
+
readonly method: string;
|
|
94
|
+
/** 1-based position in the index key (INCLUDE columns are not part of the key). */
|
|
95
|
+
readonly position: number;
|
|
96
|
+
/** The column name, or the expression text for an expression index part. */
|
|
97
|
+
readonly columnName: string;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface ForeignKeyColumnRow {
|
|
101
|
+
readonly schemaName: string;
|
|
102
|
+
readonly tableName: string;
|
|
103
|
+
readonly constraintName: string;
|
|
104
|
+
readonly position: number;
|
|
105
|
+
readonly columnName: string;
|
|
106
|
+
readonly referencedSchema: string;
|
|
107
|
+
readonly referencedTable: string;
|
|
108
|
+
readonly referencedColumn: string;
|
|
109
|
+
/** pg_constraint.confupdtype / confdeltype: one letter, see FOREIGN_KEY_ACTIONS. */
|
|
110
|
+
readonly updateRule: string;
|
|
111
|
+
readonly deleteRule: string;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export interface TriggerRow {
|
|
115
|
+
readonly schemaName: string;
|
|
116
|
+
readonly tableName: string;
|
|
117
|
+
readonly triggerName: string;
|
|
118
|
+
/** pg_trigger.tgtype: a bit set, see TRIGGER_TYPE. */
|
|
119
|
+
readonly type: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export interface RoutineRow {
|
|
123
|
+
readonly schemaName: string;
|
|
124
|
+
readonly routineName: string;
|
|
125
|
+
/** pg_proc.prokind: f function, p procedure, a aggregate, w window function. */
|
|
126
|
+
readonly kind: string;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const SYSTEM_SCHEMAS = ["pg_catalog", "information_schema", "pg_toast"];
|
|
130
|
+
// Every session gets its own pg_temp_N and pg_toast_temp_N for temporary tables.
|
|
131
|
+
const TEMPORARY_SCHEMA = /^pg_(toast_)?temp_\d+$/;
|
|
132
|
+
|
|
133
|
+
const FOREIGN_KEY_ACTIONS: Readonly<Record<string, ReferentialAction>> = {
|
|
134
|
+
a: "NO ACTION",
|
|
135
|
+
r: "RESTRICT",
|
|
136
|
+
c: "CASCADE",
|
|
137
|
+
n: "SET NULL",
|
|
138
|
+
d: "SET DEFAULT",
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
// The bits of pg_trigger.tgtype (src/include/catalog/pg_trigger.h). ROW (1)
|
|
142
|
+
// says row or statement level, which the model does not record.
|
|
143
|
+
const TRIGGER_TYPE = { before: 2, insert: 4, delete: 8, update: 16, truncate: 32, instead: 64 } as const;
|
|
144
|
+
|
|
145
|
+
// Aggregates and window functions live in pg_proc too, but they are not
|
|
146
|
+
// something a person calls like a function or a procedure, so they are left out.
|
|
147
|
+
const ROUTINE_KINDS: Readonly<Record<string, Routine["kind"]>> = { f: "function", p: "procedure" };
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The schemas to capture, sorted: the requested ones that exist (names match
|
|
151
|
+
* exactly, as PostgreSQL identifiers are case-sensitive), or every schema but
|
|
152
|
+
* the system ones when none was requested.
|
|
153
|
+
*/
|
|
154
|
+
export function capturedSchemas(schemas: readonly SchemaRow[], requested: readonly string[] | null): string[] {
|
|
155
|
+
const names = schemas.map((row) => row.schemaName);
|
|
156
|
+
const chosen = requested === null ? names.filter((name) => !isSystemSchema(name)) : names.filter((name) => requested.includes(name));
|
|
157
|
+
return [...new Set(chosen)].sort(compareText);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function buildSnapshot(catalog: PostgresCatalog): Snapshot {
|
|
161
|
+
const { database } = catalog;
|
|
162
|
+
const captured = capturedSchemas(catalog.schemas, catalog.requestedSchemas);
|
|
163
|
+
const capturedSet = new Set(captured);
|
|
164
|
+
const inCapturedSchema = (row: { schemaName: string }) => capturedSet.has(row.schemaName);
|
|
165
|
+
|
|
166
|
+
// Group every kind of row by its table once, so building each table is a lookup.
|
|
167
|
+
const columnsByTable = groupBy(catalog.columns.filter(inCapturedSchema), tableKey);
|
|
168
|
+
const indexesByTable = groupBy(catalog.indexColumns.filter(inCapturedSchema), tableKey);
|
|
169
|
+
const foreignKeysByTable = groupBy(catalog.foreignKeyColumns.filter(inCapturedSchema), tableKey);
|
|
170
|
+
const triggersByTable = groupBy(catalog.triggers.filter(inCapturedSchema), tableKey);
|
|
171
|
+
const tablesBySchema = groupBy(catalog.tables.filter(inCapturedSchema), (row) => row.schemaName);
|
|
172
|
+
const routinesBySchema = groupBy(catalog.routines.filter(inCapturedSchema), (row) => row.schemaName);
|
|
173
|
+
|
|
174
|
+
const schemas: Schema[] = captured.map((name) => ({
|
|
175
|
+
name,
|
|
176
|
+
tables: (tablesBySchema.get(name) ?? []).map((row) => {
|
|
177
|
+
const key = tableKey(row);
|
|
178
|
+
return buildTable(
|
|
179
|
+
database,
|
|
180
|
+
row,
|
|
181
|
+
columnsByTable.get(key) ?? [],
|
|
182
|
+
indexesByTable.get(key) ?? [],
|
|
183
|
+
foreignKeysByTable.get(key) ?? [],
|
|
184
|
+
triggersByTable.get(key) ?? [],
|
|
185
|
+
);
|
|
186
|
+
}),
|
|
187
|
+
routines: buildRoutines(routinesBySchema.get(name) ?? []),
|
|
188
|
+
}));
|
|
189
|
+
|
|
190
|
+
// Return the canonical order so every consumer sees the same snapshot,
|
|
191
|
+
// whatever order the server returned the rows in.
|
|
192
|
+
return sortSnapshot({
|
|
193
|
+
formatVersion: 1,
|
|
194
|
+
engine: { kind: "postgresql", version: catalog.version },
|
|
195
|
+
capturedAt: catalog.capturedAt,
|
|
196
|
+
databases: [{ name: database, schemas }],
|
|
197
|
+
warnings: [
|
|
198
|
+
...missingSchemaWarnings(database, catalog.requestedSchemas, catalog.schemas),
|
|
199
|
+
...unusableSchemaWarnings(database, catalog.schemas, capturedSet),
|
|
200
|
+
...hiddenViewDefinitionWarnings(database, schemas),
|
|
201
|
+
],
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function buildTable(
|
|
206
|
+
database: string,
|
|
207
|
+
row: TableRow,
|
|
208
|
+
columnRows: readonly ColumnRow[],
|
|
209
|
+
indexRows: readonly IndexColumnRow[],
|
|
210
|
+
foreignKeyRows: readonly ForeignKeyColumnRow[],
|
|
211
|
+
triggerRows: readonly TriggerRow[],
|
|
212
|
+
): Table {
|
|
213
|
+
// A materialized view is stored like a table (it has rows and indexes) but
|
|
214
|
+
// is defined by a SELECT, which is what a reader needs to know: the model
|
|
215
|
+
// has no separate kind for it, so it is a view with its definition.
|
|
216
|
+
const isView = row.relkind === "v" || row.relkind === "m";
|
|
217
|
+
return {
|
|
218
|
+
name: row.tableName,
|
|
219
|
+
kind: isView ? "view" : "table",
|
|
220
|
+
comment: row.comment ?? "",
|
|
221
|
+
storageEngine: null,
|
|
222
|
+
rowEstimate: rowEstimate(row),
|
|
223
|
+
columns: [...columnRows].sort((a, b) => a.position - b.position).map(buildColumn),
|
|
224
|
+
primaryKey: indexColumns(indexRows.filter((r) => r.primary)),
|
|
225
|
+
indexes: buildIndexes(indexRows.filter((r) => !r.primary)),
|
|
226
|
+
foreignKeys: buildForeignKeys(database, foreignKeyRows),
|
|
227
|
+
triggers: triggerRows.map(buildTrigger),
|
|
228
|
+
viewDefinition: isView ? row.viewDefinition : null,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function rowEstimate(row: TableRow): number | null {
|
|
233
|
+
if (row.relkind === "v") return null;
|
|
234
|
+
const estimate = Number(row.reltuples);
|
|
235
|
+
if (estimate >= 0) return Math.round(estimate);
|
|
236
|
+
// A partitioned table holds no rows itself; until it is analyzed, the sum of
|
|
237
|
+
// its partitions is the best estimate there is.
|
|
238
|
+
if (row.relkind === "p" && row.partitionRows !== null) return Math.round(Number(row.partitionRows));
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function buildColumn(row: ColumnRow): Column {
|
|
243
|
+
const generated = row.generated === "s" || row.generated === "v";
|
|
244
|
+
const identity = row.identity === "a" ? "ALWAYS" : row.identity === "d" ? "BY DEFAULT" : null;
|
|
245
|
+
return {
|
|
246
|
+
name: row.columnName,
|
|
247
|
+
position: row.position,
|
|
248
|
+
type: row.enumLabels === null ? row.dataType : `${row.dataType} enum(${row.enumLabels.map(sqlText).join(",")})`,
|
|
249
|
+
nullable: !row.notNull,
|
|
250
|
+
// An identity column has no pg_attrdef row; its default is the identity
|
|
251
|
+
// itself, written as the DDL says it, so ALWAYS and BY DEFAULT differ.
|
|
252
|
+
default: generated ? null : identity !== null ? `GENERATED ${identity} AS IDENTITY` : row.defaultExpression,
|
|
253
|
+
comment: row.comment ?? "",
|
|
254
|
+
// serial and bigserial are a sequence default, the pre-identity idiom.
|
|
255
|
+
autoIncrement: identity !== null || (row.defaultExpression?.startsWith("nextval(") ?? false),
|
|
256
|
+
generated: generated ? { expression: row.defaultExpression ?? "", stored: row.generated === "s" } : null,
|
|
257
|
+
collation: row.collation,
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function buildIndexes(rows: readonly IndexColumnRow[]): Index[] {
|
|
262
|
+
return [...groupBy(rows, (r) => r.indexName)].map(([name, parts]) => {
|
|
263
|
+
const first = parts[0]!; // groupBy never creates empty groups
|
|
264
|
+
return { name, columns: indexColumns(parts), unique: first.unique, method: first.method };
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function indexColumns(rows: readonly IndexColumnRow[]): string[] {
|
|
269
|
+
return [...rows].sort((a, b) => Number(a.position) - Number(b.position)).map((r) => r.columnName);
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function buildForeignKeys(database: string, rows: readonly ForeignKeyColumnRow[]): ForeignKey[] {
|
|
273
|
+
return [...groupBy(rows, (r) => r.constraintName)].map(([name, parts]) => {
|
|
274
|
+
const sorted = [...parts].sort((a, b) => Number(a.position) - Number(b.position));
|
|
275
|
+
const first = sorted[0]!;
|
|
276
|
+
const where = `${database}.${first.schemaName}.${first.tableName}.${name}`;
|
|
277
|
+
return {
|
|
278
|
+
name,
|
|
279
|
+
columns: sorted.map((r) => r.columnName),
|
|
280
|
+
references: {
|
|
281
|
+
database,
|
|
282
|
+
schema: first.referencedSchema,
|
|
283
|
+
table: first.referencedTable,
|
|
284
|
+
columns: sorted.map((r) => r.referencedColumn),
|
|
285
|
+
},
|
|
286
|
+
onDelete: foreignKeyAction(first.deleteRule, where),
|
|
287
|
+
onUpdate: foreignKeyAction(first.updateRule, where),
|
|
288
|
+
};
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
function foreignKeyAction(code: string, where: string): ReferentialAction {
|
|
293
|
+
const action = FOREIGN_KEY_ACTIONS[code];
|
|
294
|
+
if (action === undefined) throw new Error(`unknown foreign key action "${code}" on ${where}`);
|
|
295
|
+
return action;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function buildTrigger(row: TriggerRow): Trigger {
|
|
299
|
+
const type = Number(row.type);
|
|
300
|
+
const has = (bit: number) => (type & bit) !== 0;
|
|
301
|
+
const events: Trigger["events"][number][] = [];
|
|
302
|
+
if (has(TRIGGER_TYPE.insert)) events.push("INSERT");
|
|
303
|
+
if (has(TRIGGER_TYPE.update)) events.push("UPDATE");
|
|
304
|
+
if (has(TRIGGER_TYPE.delete)) events.push("DELETE");
|
|
305
|
+
if (has(TRIGGER_TYPE.truncate)) events.push("TRUNCATE");
|
|
306
|
+
return {
|
|
307
|
+
name: row.triggerName,
|
|
308
|
+
timing: has(TRIGGER_TYPE.instead) ? "INSTEAD OF" : has(TRIGGER_TYPE.before) ? "BEFORE" : "AFTER",
|
|
309
|
+
events,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Overloads share a name (add_item(int), add_item(text)): the model lists
|
|
314
|
+
// routines by name, so each name appears once per kind.
|
|
315
|
+
function buildRoutines(rows: readonly RoutineRow[]): Routine[] {
|
|
316
|
+
const routines = new Map<string, Routine>();
|
|
317
|
+
for (const row of rows) {
|
|
318
|
+
const kind = ROUTINE_KINDS[row.kind];
|
|
319
|
+
if (kind !== undefined) routines.set(`${kind}\u0000${row.routineName}`, { name: row.routineName, kind });
|
|
320
|
+
}
|
|
321
|
+
return [...routines.values()];
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function missingSchemaWarnings(database: string, requested: readonly string[] | null, schemas: readonly SchemaRow[]): Warning[] {
|
|
325
|
+
const existing = new Set(schemas.map((row) => row.schemaName));
|
|
326
|
+
return [...new Set(requested ?? [])]
|
|
327
|
+
.filter((name) => !existing.has(name))
|
|
328
|
+
.map((name) => ({
|
|
329
|
+
code: "schema_not_visible",
|
|
330
|
+
message: `schema "${name}" does not exist in database "${database}"`,
|
|
331
|
+
target: formatName({ database, schema: name }),
|
|
332
|
+
}));
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// pg_catalog lists every object of the database to every user, privileges or
|
|
336
|
+
// not, so nothing is hidden from the snapshot. But without USAGE on a schema
|
|
337
|
+
// no query can reach its tables, which an agent writing SQL must know.
|
|
338
|
+
function unusableSchemaWarnings(database: string, schemas: readonly SchemaRow[], captured: ReadonlySet<string>): Warning[] {
|
|
339
|
+
return schemas
|
|
340
|
+
.filter((row) => captured.has(row.schemaName) && !row.usable)
|
|
341
|
+
.map((row) => ({
|
|
342
|
+
code: "schema_not_usable",
|
|
343
|
+
message: `the user has no USAGE privilege on schema "${row.schemaName}": its structure is listed, but its tables cannot be queried with this user`,
|
|
344
|
+
target: formatName({ database, schema: row.schemaName }),
|
|
345
|
+
}));
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// pg_get_viewdef does not check privileges, so a definition is only missing
|
|
349
|
+
// when the server returned none; still, null is never presented as a SELECT.
|
|
350
|
+
function hiddenViewDefinitionWarnings(database: string, schemas: readonly Schema[]): Warning[] {
|
|
351
|
+
return schemas.flatMap((schema) =>
|
|
352
|
+
schema.tables
|
|
353
|
+
.filter((table) => table.kind === "view" && table.viewDefinition === null)
|
|
354
|
+
.map((view) => ({
|
|
355
|
+
code: "view_definition_hidden",
|
|
356
|
+
message: `definition of view "${schema.name}.${view.name}" could not be read`,
|
|
357
|
+
target: formatName({ database, schema: schema.name, table: view.name }),
|
|
358
|
+
})),
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function isSystemSchema(name: string): boolean {
|
|
363
|
+
return SYSTEM_SCHEMAS.includes(name) || TEMPORARY_SCHEMA.test(name);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** A SQL string literal: 'it''s', the way enum labels are written in DDL. */
|
|
367
|
+
function sqlText(text: string): string {
|
|
368
|
+
return `'${text.replaceAll("'", "''")}'`;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function tableKey(row: { schemaName: string; tableName: string }): string {
|
|
372
|
+
// \u0000 cannot appear in identifiers, so the key is unambiguous.
|
|
373
|
+
return `${row.schemaName}\u0000${row.tableName}`;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function groupBy<T>(items: readonly T[], keyOf: (item: T) => string): Map<string, T[]> {
|
|
377
|
+
const groups = new Map<string, T[]>();
|
|
378
|
+
for (const item of items) {
|
|
379
|
+
const key = keyOf(item);
|
|
380
|
+
const group = groups.get(key);
|
|
381
|
+
if (group === undefined) groups.set(key, [item]);
|
|
382
|
+
else group.push(item);
|
|
383
|
+
}
|
|
384
|
+
return groups;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// Plain code-unit comparison, like byName: the order never depends on the locale.
|
|
388
|
+
function compareText(a: string, b: string): number {
|
|
389
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
390
|
+
}
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { Client } from "pg";
|
|
2
|
+
|
|
3
|
+
import type { Snapshot } from "../../model/snapshot.ts";
|
|
4
|
+
import {
|
|
5
|
+
buildSnapshot,
|
|
6
|
+
capturedSchemas,
|
|
7
|
+
type ColumnRow,
|
|
8
|
+
type ForeignKeyColumnRow,
|
|
9
|
+
type IndexColumnRow,
|
|
10
|
+
type RoutineRow,
|
|
11
|
+
type SchemaRow,
|
|
12
|
+
type TableRow,
|
|
13
|
+
type TriggerRow,
|
|
14
|
+
} from "./catalog.ts";
|
|
15
|
+
|
|
16
|
+
// Reads the structure of one PostgreSQL 14+ database through pg_catalog.
|
|
17
|
+
// information_schema is not enough: it hides every object the user has no
|
|
18
|
+
// privilege on, and lacks enum labels, index methods, expression indexes and
|
|
19
|
+
// materialized views. pg_catalog lists everything to everyone.
|
|
20
|
+
//
|
|
21
|
+
// One query per kind of object for ALL captured schemas at once (= ANY($1)),
|
|
22
|
+
// never one per table, like the MySQL engine. The rows go to catalog.ts,
|
|
23
|
+
// which builds the snapshot without touching the server.
|
|
24
|
+
|
|
25
|
+
export interface PostgresConnectionOptions {
|
|
26
|
+
readonly host: string;
|
|
27
|
+
readonly port: number;
|
|
28
|
+
readonly user: string;
|
|
29
|
+
/** The resolved password. Callers resolve secret references before this point. */
|
|
30
|
+
readonly password: string;
|
|
31
|
+
/** The one database to connect to: a PostgreSQL connection never reaches another. */
|
|
32
|
+
readonly database: string;
|
|
33
|
+
/** Schemas to capture; null or omitted captures every non-system schema. */
|
|
34
|
+
readonly schemas?: readonly string[] | null;
|
|
35
|
+
readonly connectTimeoutMs?: number;
|
|
36
|
+
readonly statementTimeoutMs?: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Every setting that changes how pg_catalog functions print text the hash
|
|
40
|
+
// depends on. A role or a database can set any of them (ALTER ROLE ... SET),
|
|
41
|
+
// so heydb pins its own values instead of inheriting them:
|
|
42
|
+
// - search_path empty: format_type, pg_get_expr and pg_get_viewdef qualify
|
|
43
|
+
// every name with its schema ("public.orders"), whatever the user's path.
|
|
44
|
+
// pg_catalog is still searched first, so the queries below need no prefix.
|
|
45
|
+
// - DateStyle, TimeZone, IntervalStyle, extra_float_digits: how date, time,
|
|
46
|
+
// interval and float constants in defaults and view definitions are printed.
|
|
47
|
+
// - default_transaction_read_only: the session cannot write (see connectReadOnly).
|
|
48
|
+
const SESSION_SETTINGS: readonly (readonly [string, string])[] = [
|
|
49
|
+
["search_path", ""],
|
|
50
|
+
["DateStyle", "ISO, MDY"],
|
|
51
|
+
["TimeZone", "UTC"],
|
|
52
|
+
["IntervalStyle", "postgres"],
|
|
53
|
+
["extra_float_digits", "1"],
|
|
54
|
+
["default_transaction_read_only", "on"],
|
|
55
|
+
];
|
|
56
|
+
|
|
57
|
+
const RELATION_KINDS = ["r", "p", "f", "v", "m"];
|
|
58
|
+
|
|
59
|
+
export async function introspectPostgres(options: PostgresConnectionOptions): Promise<Snapshot> {
|
|
60
|
+
const client = await connectReadOnly(options);
|
|
61
|
+
try {
|
|
62
|
+
const capturedAt = new Date().toISOString();
|
|
63
|
+
// One read-only REPEATABLE READ transaction: every query below sees the
|
|
64
|
+
// same moment, so a table created halfway through cannot appear in one
|
|
65
|
+
// list and be missing from another.
|
|
66
|
+
await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY");
|
|
67
|
+
const version = String((await client.query("SHOW server_version")).rows[0]?.server_version ?? "");
|
|
68
|
+
const database = String((await client.query("SELECT current_database() AS name")).rows[0]?.name ?? "");
|
|
69
|
+
|
|
70
|
+
const requestedSchemas = options.schemas ?? null;
|
|
71
|
+
const schemas = (
|
|
72
|
+
await client.query<SchemaRow>(
|
|
73
|
+
`SELECT nspname AS "schemaName", has_schema_privilege(oid, 'USAGE') AS usable FROM pg_namespace`,
|
|
74
|
+
)
|
|
75
|
+
).rows;
|
|
76
|
+
const captured = capturedSchemas(schemas, requestedSchemas);
|
|
77
|
+
const empty = { tables: [], columns: [], indexColumns: [], foreignKeyColumns: [], triggers: [], routines: [] };
|
|
78
|
+
const rows = captured.length === 0 ? empty : await readCatalog(client, captured);
|
|
79
|
+
await client.query("COMMIT");
|
|
80
|
+
|
|
81
|
+
return buildSnapshot({ version, capturedAt, database, requestedSchemas, schemas, ...rows });
|
|
82
|
+
} finally {
|
|
83
|
+
await client.end();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Opens a connection whose session is read-only: INSERT, UPDATE, DDL and
|
|
89
|
+
* CALL of a writing procedure all fail, even for a superuser. A superuser
|
|
90
|
+
* could switch it off again with SET; heydb only ever sends fixed SELECT
|
|
91
|
+
* statements. Defense in depth on top of recommending a read-only role.
|
|
92
|
+
*/
|
|
93
|
+
export async function connectReadOnly(options: PostgresConnectionOptions): Promise<Client> {
|
|
94
|
+
const client = new Client({
|
|
95
|
+
host: options.host,
|
|
96
|
+
port: options.port,
|
|
97
|
+
user: options.user,
|
|
98
|
+
password: options.password,
|
|
99
|
+
database: options.database,
|
|
100
|
+
connectionTimeoutMillis: options.connectTimeoutMs ?? 10_000,
|
|
101
|
+
application_name: "heydb",
|
|
102
|
+
});
|
|
103
|
+
try {
|
|
104
|
+
await client.connect();
|
|
105
|
+
} catch (error) {
|
|
106
|
+
// A failed connect may still hold a socket; ending it lets the process exit.
|
|
107
|
+
await client.end().catch(() => {});
|
|
108
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
109
|
+
throw new Error(
|
|
110
|
+
`postgresql: cannot connect to ${options.host}:${options.port} as ${options.user} (database "${options.database}"): ${reason}`,
|
|
111
|
+
{ cause: error },
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
try {
|
|
115
|
+
const settings = [...SESSION_SETTINGS, ["statement_timeout", String(options.statementTimeoutMs ?? 60_000)] as const];
|
|
116
|
+
// set_config(name, value, false) is SET for the whole session, with the values bound as parameters.
|
|
117
|
+
// Qualified, because this one statement still runs under the role's own search_path.
|
|
118
|
+
const calls = settings.map((_, index) => `pg_catalog.set_config($${index * 2 + 1}, $${index * 2 + 2}, false)`);
|
|
119
|
+
await client.query(`SELECT ${calls.join(", ")}`, settings.flat());
|
|
120
|
+
} catch (error) {
|
|
121
|
+
// Never leave a half-configured connection open: it would keep the process alive.
|
|
122
|
+
await client.end().catch(() => {});
|
|
123
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
124
|
+
throw new Error(`postgresql: cannot open a read-only session on ${options.host}:${options.port}: ${reason}`, {
|
|
125
|
+
cause: error,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
return client;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function readCatalog(client: Client, schemas: readonly string[]) {
|
|
132
|
+
// Every query takes the captured schemas as $1 (a text[]) and, where it
|
|
133
|
+
// needs them, the relation kinds heydb lists as $2; nothing is interpolated.
|
|
134
|
+
const select = async <T>(sql: string, ...more: unknown[]): Promise<T[]> =>
|
|
135
|
+
(await client.query(sql, [schemas, ...more])).rows as T[];
|
|
136
|
+
|
|
137
|
+
// One after another, never with Promise.all: a pg client runs one query at
|
|
138
|
+
// a time anyway, and queueing a second one while the first runs is
|
|
139
|
+
// deprecated (pg 8 warns; pg 9 will refuse it).
|
|
140
|
+
//
|
|
141
|
+
// Partitions are left out (NOT relispartition): the partitioned table is
|
|
142
|
+
// the object people query, and listing each partition would repeat it.
|
|
143
|
+
const tables = await select<TableRow>(
|
|
144
|
+
`
|
|
145
|
+
SELECT n.nspname AS "schemaName", c.relname AS "tableName", c.relkind::text AS relkind,
|
|
146
|
+
c.reltuples AS reltuples,
|
|
147
|
+
CASE WHEN c.relkind = 'p' THEN (
|
|
148
|
+
SELECT sum(part.reltuples) FROM pg_partition_tree(c.oid) tree
|
|
149
|
+
JOIN pg_class part ON part.oid = tree.relid
|
|
150
|
+
WHERE tree.isleaf AND part.reltuples >= 0
|
|
151
|
+
) END AS "partitionRows",
|
|
152
|
+
obj_description(c.oid, 'pg_class') AS comment,
|
|
153
|
+
CASE WHEN c.relkind IN ('v', 'm') THEN pg_get_viewdef(c.oid, true) END AS "viewDefinition"
|
|
154
|
+
FROM pg_class c
|
|
155
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
156
|
+
WHERE n.nspname = ANY($1) AND c.relkind = ANY($2) AND NOT c.relispartition`,
|
|
157
|
+
RELATION_KINDS,
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
// The collation is reported only when it differs from the type's own,
|
|
161
|
+
// i.e. when the column sets one explicitly.
|
|
162
|
+
const columns = await select<ColumnRow>(
|
|
163
|
+
`
|
|
164
|
+
SELECT n.nspname AS "schemaName", c.relname AS "tableName", a.attname AS "columnName",
|
|
165
|
+
a.attnum AS position, format_type(a.atttypid, a.atttypmod) AS "dataType",
|
|
166
|
+
a.attnotnull AS "notNull", pg_get_expr(d.adbin, d.adrelid) AS "defaultExpression",
|
|
167
|
+
a.attidentity::text AS identity, a.attgenerated::text AS generated,
|
|
168
|
+
col_description(c.oid, a.attnum) AS comment,
|
|
169
|
+
CASE WHEN a.attcollation <> t.typcollation THEN co.collname::text END AS collation,
|
|
170
|
+
CASE WHEN t.typtype = 'e' THEN ARRAY(
|
|
171
|
+
SELECT e.enumlabel::text FROM pg_enum e WHERE e.enumtypid = t.oid ORDER BY e.enumsortorder
|
|
172
|
+
) END AS "enumLabels"
|
|
173
|
+
FROM pg_attribute a
|
|
174
|
+
JOIN pg_class c ON c.oid = a.attrelid
|
|
175
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
176
|
+
JOIN pg_type t ON t.oid = a.atttypid
|
|
177
|
+
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum
|
|
178
|
+
LEFT JOIN pg_collation co ON co.oid = a.attcollation
|
|
179
|
+
WHERE n.nspname = ANY($1) AND c.relkind = ANY($2) AND NOT c.relispartition
|
|
180
|
+
AND a.attnum > 0 AND NOT a.attisdropped`,
|
|
181
|
+
RELATION_KINDS,
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
// indkey is 0-based and holds 0 for an expression part, whose text
|
|
185
|
+
// pg_get_indexdef prints. Only key columns, not INCLUDE ones.
|
|
186
|
+
const indexColumns = await select<IndexColumnRow>(`
|
|
187
|
+
SELECT n.nspname AS "schemaName", c.relname AS "tableName", ic.relname AS "indexName",
|
|
188
|
+
x.indisprimary AS "primary", x.indisunique AS "unique", am.amname AS method,
|
|
189
|
+
k.ord::int AS position,
|
|
190
|
+
CASE WHEN x.indkey[k.ord - 1] = 0 THEN pg_get_indexdef(x.indexrelid, k.ord::int, true)
|
|
191
|
+
ELSE a.attname::text END AS "columnName"
|
|
192
|
+
FROM pg_index x
|
|
193
|
+
JOIN pg_class c ON c.oid = x.indrelid
|
|
194
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
195
|
+
JOIN pg_class ic ON ic.oid = x.indexrelid
|
|
196
|
+
JOIN pg_am am ON am.oid = ic.relam
|
|
197
|
+
CROSS JOIN LATERAL generate_series(1, x.indnkeyatts) AS k(ord)
|
|
198
|
+
LEFT JOIN pg_attribute a ON a.attrelid = x.indrelid AND a.attnum = x.indkey[k.ord - 1]
|
|
199
|
+
WHERE n.nspname = ANY($1) AND NOT c.relispartition`);
|
|
200
|
+
|
|
201
|
+
// conkey and confkey pair up position by position. A foreign key to a
|
|
202
|
+
// partitioned table also gets one clone per partition (conparentid <> 0).
|
|
203
|
+
const foreignKeyColumns = await select<ForeignKeyColumnRow>(`
|
|
204
|
+
SELECT n.nspname AS "schemaName", c.relname AS "tableName", con.conname AS "constraintName",
|
|
205
|
+
k.ord::int AS position, a.attname AS "columnName",
|
|
206
|
+
rn.nspname AS "referencedSchema", rc.relname AS "referencedTable", ra.attname AS "referencedColumn",
|
|
207
|
+
con.confupdtype::text AS "updateRule", con.confdeltype::text AS "deleteRule"
|
|
208
|
+
FROM pg_constraint con
|
|
209
|
+
JOIN pg_class c ON c.oid = con.conrelid
|
|
210
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
211
|
+
JOIN pg_class rc ON rc.oid = con.confrelid
|
|
212
|
+
JOIN pg_namespace rn ON rn.oid = rc.relnamespace
|
|
213
|
+
CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(attnum, refattnum, ord)
|
|
214
|
+
JOIN pg_attribute a ON a.attrelid = con.conrelid AND a.attnum = k.attnum
|
|
215
|
+
JOIN pg_attribute ra ON ra.attrelid = con.confrelid AND ra.attnum = k.refattnum
|
|
216
|
+
WHERE n.nspname = ANY($1) AND con.contype = 'f' AND con.conparentid = 0 AND NOT c.relispartition`);
|
|
217
|
+
|
|
218
|
+
// Internal triggers are the ones PostgreSQL creates to enforce foreign keys.
|
|
219
|
+
const triggers = await select<TriggerRow>(`
|
|
220
|
+
SELECT n.nspname AS "schemaName", c.relname AS "tableName", t.tgname AS "triggerName", t.tgtype AS type
|
|
221
|
+
FROM pg_trigger t
|
|
222
|
+
JOIN pg_class c ON c.oid = t.tgrelid
|
|
223
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
224
|
+
WHERE n.nspname = ANY($1) AND NOT t.tgisinternal AND NOT c.relispartition`);
|
|
225
|
+
|
|
226
|
+
// Functions that belong to an extension (citext, PostGIS…) are part of
|
|
227
|
+
// that extension, not of the application's schema: PostGIS alone adds
|
|
228
|
+
// hundreds. pg_depend records them with deptype 'e'.
|
|
229
|
+
const routines = await select<RoutineRow>(`
|
|
230
|
+
SELECT n.nspname AS "schemaName", p.proname AS "routineName", p.prokind::text AS kind
|
|
231
|
+
FROM pg_proc p
|
|
232
|
+
JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
233
|
+
WHERE n.nspname = ANY($1)
|
|
234
|
+
AND NOT EXISTS (
|
|
235
|
+
SELECT 1 FROM pg_depend dep
|
|
236
|
+
WHERE dep.classid = 'pg_proc'::regclass AND dep.objid = p.oid AND dep.deptype = 'e'
|
|
237
|
+
)`);
|
|
238
|
+
|
|
239
|
+
return { tables, columns, indexColumns, foreignKeyColumns, triggers, routines };
|
|
240
|
+
}
|