@pikku/cli 0.12.89 → 0.12.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/console-app/assets/{index-BaOGIr2O.js → index-C5Bd44e4.js} +149 -149
- package/console-app/index.html +1 -1
- package/dist/.pikku/agent/pikku-agent-types.gen.d.ts +1 -1
- package/dist/.pikku/channel/pikku-channel-types.gen.d.ts +1 -1
- package/dist/.pikku/channel/pikku-channel-types.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli-channel.js +21 -1
- package/dist/.pikku/cli/pikku-cli-client.gen.d.ts +1 -1
- package/dist/.pikku/cli/pikku-cli-client.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli-contracts-meta.gen.d.ts +1 -1
- package/dist/.pikku/cli/pikku-cli-contracts-meta.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli-contracts-meta.gen.json +6 -1
- package/dist/.pikku/cli/pikku-cli-types.gen.d.ts +1 -1
- package/dist/.pikku/cli/pikku-cli-types.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli-wirings-meta.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli-wirings-meta.gen.json +30 -1
- package/dist/.pikku/cli/pikku-cli-wirings.gen.d.ts +1 -1
- package/dist/.pikku/cli/pikku-cli-wirings.gen.js +1 -1
- package/dist/.pikku/cli/pikku-cli.gen.d.ts +1 -1
- package/dist/.pikku/cli/pikku-cli.gen.js +1 -1
- package/dist/.pikku/console/pikku-node-types.gen.d.ts +1 -1
- package/dist/.pikku/function/pikku-function-types.gen.d.ts +1 -1
- package/dist/.pikku/function/pikku-function-types.gen.js +1 -1
- package/dist/.pikku/function/pikku-functions-meta.gen.js +1 -1
- package/dist/.pikku/function/pikku-functions-meta.gen.json +64 -4
- package/dist/.pikku/function/pikku-functions.gen.js +9 -1
- package/dist/.pikku/http/pikku-http-types.gen.d.ts +1 -1
- package/dist/.pikku/http/pikku-http-types.gen.js +1 -1
- package/dist/.pikku/mcp/pikku-mcp-types.gen.d.ts +1 -1
- package/dist/.pikku/mcp/pikku-mcp-types.gen.js +1 -1
- package/dist/.pikku/pikku-bootstrap.gen.d.ts +1 -1
- package/dist/.pikku/pikku-bootstrap.gen.js +1 -1
- package/dist/.pikku/pikku-meta-service.gen.d.ts +1 -1
- package/dist/.pikku/pikku-meta-service.gen.js +1 -1
- package/dist/.pikku/pikku-services.gen.d.ts +1 -1
- package/dist/.pikku/pikku-types.gen.d.ts +1 -1
- package/dist/.pikku/pikku-types.gen.js +1 -1
- package/dist/.pikku/queue/pikku-queue-types.gen.d.ts +1 -1
- package/dist/.pikku/queue/pikku-queue-types.gen.js +1 -1
- package/dist/.pikku/queue/pikku-queue-workers-wirings-meta.gen.js +1 -1
- package/dist/.pikku/queue/pikku-queue-workers-wirings.gen.d.ts +1 -1
- package/dist/.pikku/queue/pikku-queue-workers-wirings.gen.js +1 -1
- package/dist/.pikku/rpc/pikku-rpc-wirings-meta.internal.gen.js +1 -1
- package/dist/.pikku/rpc/pikku-rpc-wirings-meta.internal.gen.json +4 -0
- package/dist/.pikku/scheduler/pikku-scheduler-types.gen.d.ts +1 -1
- package/dist/.pikku/scheduler/pikku-scheduler-types.gen.js +1 -1
- package/dist/.pikku/schemas/register.gen.js +9 -1
- package/dist/.pikku/schemas/schemas/DbBaselineInput.schema.json +1 -0
- package/dist/.pikku/schemas/schemas/DbCheckInput.schema.json +1 -0
- package/dist/.pikku/schemas/schemas/DbCodegenInput.schema.json +1 -0
- package/dist/.pikku/schemas/schemas/DbExportInput.schema.json +1 -0
- package/dist/.pikku/schemas/schemas/FabricValidateInput.schema.json +1 -1
- package/dist/.pikku/scopes/pikku-scope-types.gen.d.ts +1 -1
- package/dist/.pikku/scopes/pikku-scope-types.gen.js +1 -1
- package/dist/.pikku/scopes/pikku-scopes.gen.d.ts +1 -1
- package/dist/.pikku/secrets/pikku-secret-types.gen.d.ts +1 -1
- package/dist/.pikku/secrets/pikku-secret-types.gen.js +1 -1
- package/dist/.pikku/secrets/pikku-secrets.gen.d.ts +1 -1
- package/dist/.pikku/secrets/pikku-secrets.gen.js +1 -1
- package/dist/.pikku/trigger/pikku-trigger-types.gen.d.ts +1 -1
- package/dist/.pikku/trigger/pikku-trigger-types.gen.js +1 -1
- package/dist/.pikku/variables/pikku-variable-types.gen.d.ts +1 -1
- package/dist/.pikku/variables/pikku-variable-types.gen.js +1 -1
- package/dist/.pikku/variables/pikku-variables.gen.d.ts +1 -1
- package/dist/.pikku/variables/pikku-variables.gen.js +1 -1
- package/dist/.pikku/workflow/pikku-workflow-types.gen.d.ts +1 -1
- package/dist/.pikku/workflow/pikku-workflow-types.gen.js +1 -1
- package/dist/.pikku/workflow/pikku-workflow-wirings-meta.gen.js +1 -1
- package/dist/.pikku/workflow/pikku-workflow-wirings.gen.js +1 -1
- package/dist/bin/pikku-bin.mjs +2 -2
- package/dist/src/cli.wiring.js +20 -0
- package/dist/src/fabric/fabric-commands.d.ts +9 -3
- package/dist/src/fabric/fabric-commands.js +6 -0
- package/dist/src/fabric/functions/validate.function.d.ts +15 -5
- package/dist/src/fabric/functions/validate.function.js +69 -16
- package/dist/src/fabric/lib/config.d.ts +11 -1
- package/dist/src/fabric/lib/config.js +39 -1
- package/dist/src/fabric/lib/frontend-typecheck.d.ts +46 -0
- package/dist/src/fabric/lib/frontend-typecheck.js +139 -0
- package/dist/src/functions/commands/db-baseline.d.ts +13 -0
- package/dist/src/functions/commands/db-baseline.js +48 -0
- package/dist/src/functions/commands/db-check.d.ts +12 -0
- package/dist/src/functions/commands/db-check.js +64 -0
- package/dist/src/functions/commands/db-codegen.d.ts +11 -0
- package/dist/src/functions/commands/db-codegen.js +39 -0
- package/dist/src/functions/commands/db-export.d.ts +12 -0
- package/dist/src/functions/commands/db-export.js +31 -0
- package/dist/src/functions/commands/db-generate.js +19 -26
- package/dist/src/functions/db/db-migrator.d.ts +35 -4
- package/dist/src/functions/db/db-migrator.js +67 -10
- package/dist/src/functions/db/local-db.d.ts +235 -13
- package/dist/src/functions/db/local-db.js +430 -45
- package/dist/src/functions/db/postgres/postgres-introspector.js +19 -1
- package/dist/src/functions/db/postgres/postgres-migrator.d.ts +1 -0
- package/dist/src/functions/db/postgres/postgres-migrator.js +4 -1
- package/dist/src/functions/db/sqlite/sqlite-introspector.js +2 -1
- package/dist/src/functions/db/sqlite/sqlite-migrator.d.ts +1 -0
- package/dist/src/functions/db/sqlite/sqlite-migrator.js +6 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +3 -3
- package/skills/pikku-i18n/SKILL.md +53 -6
|
@@ -4,7 +4,8 @@ import { createRequire } from 'node:module';
|
|
|
4
4
|
import { runInNewContext } from 'node:vm';
|
|
5
5
|
import { transformSync } from 'esbuild';
|
|
6
6
|
import { CamelCasePlugin, Kysely, PostgresDialect } from 'kysely';
|
|
7
|
-
import { migrate } from './db-migrator.js';
|
|
7
|
+
import { migrate, baselineMigrations, } from './db-migrator.js';
|
|
8
|
+
import { applyPikkuSchemas, compilePikkuSchemas, pikkuSchemas, resolveRequirements, } from '@pikku/kysely';
|
|
8
9
|
import { loadAuthOptions, getAuthMigrations } from './better-auth-schema.js';
|
|
9
10
|
import { generateSchemaTypes } from './db-codegen.js';
|
|
10
11
|
import { generateZodTypes } from './zod-codegen.js';
|
|
@@ -163,16 +164,18 @@ async function loadCoercionPlugin(coercionFile) {
|
|
|
163
164
|
return undefined;
|
|
164
165
|
}
|
|
165
166
|
}
|
|
166
|
-
export async function migrateAndCodegen(resolved) {
|
|
167
|
+
export async function migrateAndCodegen(resolved, options = {}) {
|
|
167
168
|
let migrateResult;
|
|
168
169
|
let codegenResult;
|
|
169
170
|
// Compile any authored db/annotations.ts → sidecar BEFORE codegen so edits
|
|
170
171
|
// reflect in a single `db migrate` (codegen reads the sidecar).
|
|
171
172
|
compileClassifications(resolved.classificationsFile, resolved.classificationsGenJsonFile);
|
|
172
173
|
if (resolved.dialect === 'sqlite') {
|
|
173
|
-
mkdirSync(dirname(resolved.dbFile), { recursive: true });
|
|
174
174
|
const runtime = await loadSqliteRuntime();
|
|
175
|
-
|
|
175
|
+
if (!options.scratch) {
|
|
176
|
+
mkdirSync(dirname(resolved.dbFile), { recursive: true });
|
|
177
|
+
}
|
|
178
|
+
const db = runtime.open(options.scratch ? ':memory:' : resolved.dbFile);
|
|
176
179
|
try {
|
|
177
180
|
const executor = new SqliteMigrationExecutor(db);
|
|
178
181
|
migrateResult = await migrate(executor, resolved.migrationsDir);
|
|
@@ -194,7 +197,10 @@ export async function migrateAndCodegen(resolved) {
|
|
|
194
197
|
}
|
|
195
198
|
}
|
|
196
199
|
else {
|
|
197
|
-
|
|
200
|
+
const withClient = options.scratch
|
|
201
|
+
? (_r, run) => withScratchPostgresDatabase(run)
|
|
202
|
+
: withPostgresClient;
|
|
203
|
+
await withClient(resolved, async (client) => {
|
|
198
204
|
const introspector = new PostgresIntrospector(client);
|
|
199
205
|
await introspector.connect();
|
|
200
206
|
try {
|
|
@@ -436,36 +442,44 @@ export async function createKysely(resolved) {
|
|
|
436
442
|
async function introspectorToMap(intro) {
|
|
437
443
|
const map = new Map();
|
|
438
444
|
for (const [table, cols] of await intro.getAllColumns()) {
|
|
439
|
-
map.set(table, new
|
|
445
|
+
map.set(table, new Map(cols.map((c) => [c.name, c])));
|
|
440
446
|
}
|
|
441
447
|
return map;
|
|
442
448
|
}
|
|
443
449
|
function diffSchemas(desired, actual) {
|
|
444
450
|
const missingTables = [];
|
|
445
451
|
const missingColumns = [];
|
|
446
|
-
const findSchemaQualifiedMatch = (table) => {
|
|
447
|
-
if (table.includes('.'))
|
|
448
|
-
return undefined;
|
|
449
|
-
const matches = [...actual.entries()].filter(([actualTable]) => {
|
|
450
|
-
const parts = actualTable.split('.');
|
|
451
|
-
return parts.length === 2 && parts[1] === table;
|
|
452
|
-
});
|
|
453
|
-
if (matches.length !== 1)
|
|
454
|
-
return undefined;
|
|
455
|
-
return matches[0][1];
|
|
456
|
-
};
|
|
457
452
|
for (const [table, cols] of desired) {
|
|
458
|
-
const actualCols = actual.get(table) ??
|
|
453
|
+
const actualCols = actual.get(table) ?? schemaQualifiedMatch(actual, table);
|
|
459
454
|
if (!actualCols) {
|
|
460
455
|
missingTables.push(table);
|
|
461
456
|
continue;
|
|
462
457
|
}
|
|
463
|
-
const missing = [...cols].filter((c) => !actualCols.has(c));
|
|
458
|
+
const missing = [...cols.keys()].filter((c) => !actualCols.has(c));
|
|
464
459
|
if (missing.length)
|
|
465
460
|
missingColumns.push({ table, columns: missing });
|
|
466
461
|
}
|
|
467
462
|
return { missingTables, missingColumns };
|
|
468
463
|
}
|
|
464
|
+
/**
|
|
465
|
+
* The one table in `actual` whose name matches `table` once its schema is
|
|
466
|
+
* dropped, if there is exactly one.
|
|
467
|
+
*
|
|
468
|
+
* A migration that names no schema lands in whichever one is default, so a bare
|
|
469
|
+
* desired name legitimately matches a qualified table. Only when it is
|
|
470
|
+
* unambiguous: two candidates mean the shadowing bug, not a match.
|
|
471
|
+
*/
|
|
472
|
+
function schemaQualifiedMatch(actual, table) {
|
|
473
|
+
if (table.includes('.'))
|
|
474
|
+
return undefined;
|
|
475
|
+
const matches = [...actual.entries()].filter(([actualTable]) => {
|
|
476
|
+
const parts = actualTable.split('.');
|
|
477
|
+
return parts.length === 2 && parts[1] === table;
|
|
478
|
+
});
|
|
479
|
+
if (matches.length !== 1)
|
|
480
|
+
return undefined;
|
|
481
|
+
return matches[0][1];
|
|
482
|
+
}
|
|
469
483
|
function isPostgresAuthDatabase(options) {
|
|
470
484
|
return options.database?.type === 'postgres';
|
|
471
485
|
}
|
|
@@ -476,7 +490,7 @@ function isPostgresAuthDatabase(options) {
|
|
|
476
490
|
// target server. Creating a real database via `CREATE DATABASE` would require
|
|
477
491
|
// elevated privileges that application roles (correctly) don't have, which made
|
|
478
492
|
// `pikku db migrate` fail against managed/locked-down Postgres (error 42501).
|
|
479
|
-
async function withScratchPostgresDatabase(
|
|
493
|
+
async function withScratchPostgresDatabase(run) {
|
|
480
494
|
const scratchDb = await createEmbeddedPostgres();
|
|
481
495
|
try {
|
|
482
496
|
return await run(pgliteAsClient(scratchDb));
|
|
@@ -497,8 +511,8 @@ async function postgresDatabaseToMap(client) {
|
|
|
497
511
|
await intro.close();
|
|
498
512
|
}
|
|
499
513
|
}
|
|
500
|
-
async function desiredPostgresAuthSchema(
|
|
501
|
-
return withScratchPostgresDatabase(
|
|
514
|
+
async function desiredPostgresAuthSchema(rootDir, srcDirectories, logger) {
|
|
515
|
+
return withScratchPostgresDatabase(async (scratchDb) => {
|
|
502
516
|
// The scratch DB is always an embedded PGlite instance (see
|
|
503
517
|
// withScratchPostgresDatabase), so drive Better Auth's migration codegen
|
|
504
518
|
// through the PGlite-backed Kysely regardless of how the app DB is
|
|
@@ -544,7 +558,7 @@ export async function desiredAuthSchema(resolved, rootDir, srcDirectories, logge
|
|
|
544
558
|
if (resolved.dialect !== 'postgres') {
|
|
545
559
|
throw new Error('Better Auth database.type is postgres, but the resolved app database is not postgres.');
|
|
546
560
|
}
|
|
547
|
-
return desiredPostgresAuthSchema(
|
|
561
|
+
return desiredPostgresAuthSchema(rootDir, srcDirectories, logger);
|
|
548
562
|
}
|
|
549
563
|
const { runMigrations, compileMigrations } = await getAuthMigrations(options);
|
|
550
564
|
await runMigrations();
|
|
@@ -556,6 +570,99 @@ export async function desiredAuthSchema(resolved, rootDir, srcDirectories, logge
|
|
|
556
570
|
db.close();
|
|
557
571
|
}
|
|
558
572
|
}
|
|
573
|
+
/**
|
|
574
|
+
* Run Better Auth's own migrator against `kysely`, if the project configures it.
|
|
575
|
+
*
|
|
576
|
+
* Separate from `desiredAuthSchema` because the auth tables are a prerequisite
|
|
577
|
+
* of the runtime ones, not just a peer of them: both have to land in the same
|
|
578
|
+
* throwaway database, in that order, for the scope tables' foreign keys onto
|
|
579
|
+
* `user.id` to resolve.
|
|
580
|
+
*/
|
|
581
|
+
async function applyAuthSchema(kysely, rootDir, srcDirectories, logger) {
|
|
582
|
+
const options = await loadAuthOptions({
|
|
583
|
+
rootDir,
|
|
584
|
+
srcDirectories,
|
|
585
|
+
kysely,
|
|
586
|
+
logger,
|
|
587
|
+
});
|
|
588
|
+
if (!options)
|
|
589
|
+
return false;
|
|
590
|
+
const { runMigrations } = await getAuthMigrations(options);
|
|
591
|
+
await runMigrations();
|
|
592
|
+
return true;
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* The tables `@pikku/kysely`'s runtime services need, as declared.
|
|
596
|
+
*
|
|
597
|
+
* Materialized the same way the auth schema is: applied to a throwaway database
|
|
598
|
+
* and introspected, so one declaration answers both "what should exist" and
|
|
599
|
+
* "what SQL creates it" without a hand-written per-dialect copy.
|
|
600
|
+
*
|
|
601
|
+
* Auth goes into the same database first, because it is a prerequisite — the
|
|
602
|
+
* scope tables grant to a user, so they reference the table Better Auth owns.
|
|
603
|
+
* Its tables are subtracted from the result: they are covered by auth's own
|
|
604
|
+
* source, and counting them twice would have `db generate` write them twice.
|
|
605
|
+
*
|
|
606
|
+
* A project with no auth configured is not an error here. It genuinely has no
|
|
607
|
+
* scope tables, so the schemas that wanted them are left out and returned in
|
|
608
|
+
* `skipped` — reported rather than dropped, because the tables they would have
|
|
609
|
+
* recognised now have nothing to explain them.
|
|
610
|
+
*/
|
|
611
|
+
export async function desiredRuntimeSchema(resolved, rootDir, srcDirectories, logger) {
|
|
612
|
+
const skipped = [];
|
|
613
|
+
const collect = async (db, introspect) => {
|
|
614
|
+
await applyAuthSchema(db, rootDir, srcDirectories, logger);
|
|
615
|
+
const before = await introspect();
|
|
616
|
+
const { types, unmet } = await resolveRequirements(db);
|
|
617
|
+
for (const { schema, requirement } of unmet) {
|
|
618
|
+
skipped.push({
|
|
619
|
+
schema: schema.name,
|
|
620
|
+
requires: `${requirement.table}.${requirement.column}`,
|
|
621
|
+
owner: requirement.owner,
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
const unavailable = new Set(unmet.map(({ schema }) => schema.name));
|
|
625
|
+
const schemas = pikkuSchemas.filter((s) => !unavailable.has(s.name));
|
|
626
|
+
await applyPikkuSchemas(db, schemas);
|
|
627
|
+
const after = await introspect();
|
|
628
|
+
for (const table of before.keys())
|
|
629
|
+
after.delete(table);
|
|
630
|
+
return { tables: after, schemas, types };
|
|
631
|
+
};
|
|
632
|
+
if (resolved.dialect === 'sqlite') {
|
|
633
|
+
const runtime = await loadSqliteRuntime();
|
|
634
|
+
const db = runtime.open(':memory:');
|
|
635
|
+
try {
|
|
636
|
+
const kysely = createSqliteKysely({ db, camelCase: true });
|
|
637
|
+
const { tables, schemas, types } = await collect(kysely, () => introspectorToMap(new SqliteIntrospector(db)));
|
|
638
|
+
return {
|
|
639
|
+
tables,
|
|
640
|
+
sql: compilePikkuSchemas(kysely, schemas, types),
|
|
641
|
+
skipped,
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
finally {
|
|
645
|
+
db.close();
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
return withScratchPostgresDatabase(async (db) => {
|
|
649
|
+
const kysely = createPGliteKysely({
|
|
650
|
+
db: db.__pglite,
|
|
651
|
+
camelCase: true,
|
|
652
|
+
});
|
|
653
|
+
try {
|
|
654
|
+
const { tables, schemas, types } = await collect(kysely, () => postgresDatabaseToMap(db));
|
|
655
|
+
return {
|
|
656
|
+
tables,
|
|
657
|
+
sql: compilePikkuSchemas(kysely, schemas, types),
|
|
658
|
+
skipped,
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
finally {
|
|
662
|
+
await kysely.destroy();
|
|
663
|
+
}
|
|
664
|
+
});
|
|
665
|
+
}
|
|
559
666
|
export async function introspectSchema(resolved) {
|
|
560
667
|
if (resolved.dialect === 'sqlite') {
|
|
561
668
|
const runtime = await loadSqliteRuntime();
|
|
@@ -589,12 +696,93 @@ async function coveredSqliteSchema(migrationsDir) {
|
|
|
589
696
|
db.close();
|
|
590
697
|
}
|
|
591
698
|
}
|
|
592
|
-
async function coveredPostgresSchema(
|
|
593
|
-
return withScratchPostgresDatabase(
|
|
699
|
+
async function coveredPostgresSchema(migrationsDir) {
|
|
700
|
+
return withScratchPostgresDatabase(async (client) => {
|
|
594
701
|
await migrate(new PostgresMigrationExecutor(client), migrationsDir);
|
|
595
702
|
return postgresDatabaseToMap(client);
|
|
596
703
|
});
|
|
597
704
|
}
|
|
705
|
+
/**
|
|
706
|
+
* Compare the schema the migration files define against the one the configured
|
|
707
|
+
* database actually has.
|
|
708
|
+
*
|
|
709
|
+
* The two halves are asymmetric and must stay that way. Something missing from
|
|
710
|
+
* the database is a database that is behind — the fix only ever adds, so it is
|
|
711
|
+
* safe to automate. Something present in the database but absent from the
|
|
712
|
+
* migrations is a table nobody wrote down: a runtime that created its own at
|
|
713
|
+
* boot, or the remains of a reverted branch. Dropping those is how data gets
|
|
714
|
+
* lost, so they are reported and never acted on.
|
|
715
|
+
*
|
|
716
|
+
* The pikku runtime declares tables of its own, and they are used here to
|
|
717
|
+
* recognise rather than to require. A project that never constructs the
|
|
718
|
+
* workflow or AI services should not be told it is missing their tables, so
|
|
719
|
+
* absence is not a finding. Presence is: a runtime table in the database that
|
|
720
|
+
* no migration creates gets reported as such, separately from the genuinely
|
|
721
|
+
* unexplained ones, because for those the remedy is known — `db generate`
|
|
722
|
+
* writes them down.
|
|
723
|
+
*/
|
|
724
|
+
export async function computeSchemaDrift(resolved, rootDir, srcDirectories, logger) {
|
|
725
|
+
const covered = resolved.dialect === 'sqlite'
|
|
726
|
+
? await coveredSqliteSchema(resolved.migrationsDir)
|
|
727
|
+
: await coveredPostgresSchema(resolved.migrationsDir);
|
|
728
|
+
const actual = await introspectSchema(resolved);
|
|
729
|
+
const { missingTables, missingColumns } = diffSchemas(covered, actual);
|
|
730
|
+
// Compare on the full name. A migration that names no schema lands in
|
|
731
|
+
// whichever one is default, so a BARE covered name still matches a qualified
|
|
732
|
+
// table — but a QUALIFIED one must match exactly. Relaxing that second case
|
|
733
|
+
// hides the failure this is here to catch: a second copy of a table in the
|
|
734
|
+
// wrong schema (`public.orders` shadowing `app.orders`) would otherwise look
|
|
735
|
+
// like the table the migrations created.
|
|
736
|
+
const coveredFull = new Set(covered.keys());
|
|
737
|
+
const coveredBare = new Set([...covered.keys()].filter((t) => !t.includes('.')));
|
|
738
|
+
const unrecorded = [...actual.keys()].filter((t) => !coveredFull.has(t) && !coveredBare.has(t.split('.').pop()));
|
|
739
|
+
const runtime = await desiredRuntimeSchema(resolved, rootDir, srcDirectories, logger);
|
|
740
|
+
const runtimeTables = unrecorded.filter((t) => runtime.tables.has(t.split('.').pop()));
|
|
741
|
+
const extraTables = unrecorded.filter((t) => !runtimeTables.includes(t));
|
|
742
|
+
return {
|
|
743
|
+
missingTables,
|
|
744
|
+
missingColumns,
|
|
745
|
+
runtimeTables,
|
|
746
|
+
extraTables,
|
|
747
|
+
skippedRuntimeSchemas: runtime.skipped,
|
|
748
|
+
inSync: missingTables.length === 0 && missingColumns.length === 0,
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Record the pending migrations as applied, for a database that already has
|
|
753
|
+
* what they describe.
|
|
754
|
+
*
|
|
755
|
+
* The situation this exists for: a runtime created its tables at boot, and the
|
|
756
|
+
* migration writing them down was authored afterwards. Every existing
|
|
757
|
+
* deployment already has those tables, so running that migration fails; not
|
|
758
|
+
* running it leaves the history claiming the schema is something else.
|
|
759
|
+
*
|
|
760
|
+
* Guarded by the same question `db check` answers. If the database is behind
|
|
761
|
+
* its migrations in any way then the premise is false — the tables are not all
|
|
762
|
+
* there — and recording them would hide a real gap behind a history that says
|
|
763
|
+
* everything is applied. So it refuses and hands back the drift.
|
|
764
|
+
*/
|
|
765
|
+
export async function baseline(resolved, rootDir, srcDirectories, logger) {
|
|
766
|
+
const drift = await computeSchemaDrift(resolved, rootDir, srcDirectories, logger);
|
|
767
|
+
if (!drift.inSync)
|
|
768
|
+
return { status: 'behind', drift };
|
|
769
|
+
if (resolved.dialect === 'sqlite') {
|
|
770
|
+
const runtime = await loadSqliteRuntime();
|
|
771
|
+
mkdirSync(dirname(resolved.dbFile), { recursive: true });
|
|
772
|
+
const db = runtime.open(resolved.dbFile);
|
|
773
|
+
try {
|
|
774
|
+
const recorded = await baselineMigrations(new SqliteMigrationExecutor(db), resolved.migrationsDir);
|
|
775
|
+
return { status: 'recorded', recorded };
|
|
776
|
+
}
|
|
777
|
+
finally {
|
|
778
|
+
db.close();
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
return withPostgresClient(resolved, async (client) => {
|
|
782
|
+
const recorded = await baselineMigrations(new PostgresMigrationExecutor(client), resolved.migrationsDir);
|
|
783
|
+
return { status: 'recorded', recorded };
|
|
784
|
+
});
|
|
785
|
+
}
|
|
598
786
|
export async function computeAuthDrift(resolved, rootDir, srcDirectories, logger) {
|
|
599
787
|
const desired = await desiredAuthSchema(resolved, rootDir, srcDirectories, logger);
|
|
600
788
|
if (!desired) {
|
|
@@ -630,24 +818,221 @@ function nextMigrationFile(migrationsDir, label) {
|
|
|
630
818
|
const num = String(max + 1).padStart(4, '0');
|
|
631
819
|
return join(migrationsDir, `${num}-${label}.sql`);
|
|
632
820
|
}
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
821
|
+
// ─── The addon schema channel ────────────────────────────────────────────────
|
|
822
|
+
/**
|
|
823
|
+
* Where an addon publishes its schema, relative to its package root.
|
|
824
|
+
*
|
|
825
|
+
* One more channel beside `.pikku/function`, `.pikku/scopes` and the rest, so a
|
|
826
|
+
* consumer reaches it the same way they do: through the package name, never a
|
|
827
|
+
* path into somebody else's tree.
|
|
828
|
+
*/
|
|
829
|
+
const ADDON_DB_ARTIFACT = '.pikku/db/pikku-db-meta.gen.json';
|
|
830
|
+
const serializeSchemaMap = (tables) => Object.fromEntries([...tables].map(([table, columns]) => [table, [...columns.values()]]));
|
|
831
|
+
const deserializeSchemaMap = (tables) => new Map(Object.entries(tables).map(([table, columns]) => [
|
|
832
|
+
table,
|
|
833
|
+
new Map(columns.map((column) => [column.name, column])),
|
|
834
|
+
]));
|
|
835
|
+
const concatMigrations = (migrationsDir) => readdirSync(migrationsDir)
|
|
836
|
+
.filter((f) => f.endsWith('.sql'))
|
|
837
|
+
.sort()
|
|
838
|
+
.map((f) => readFileSync(join(migrationsDir, f), 'utf8').trim())
|
|
839
|
+
.join('\n\n');
|
|
840
|
+
/**
|
|
841
|
+
* Materialize this package's own migrations into something a consumer can use.
|
|
842
|
+
*
|
|
843
|
+
* The producer half of the addon channel, and the same trick the auth and
|
|
844
|
+
* runtime sources use: run the SQL into a throwaway database and introspect it,
|
|
845
|
+
* so the artifact answers both "what must exist" and "what creates it" without
|
|
846
|
+
* a second, hand-written description that drifts.
|
|
847
|
+
*
|
|
848
|
+
* Every dialect the package has migrations for is exported, not just the one it
|
|
849
|
+
* happens to be configured against — an addon is published once and consumed by
|
|
850
|
+
* projects on either engine.
|
|
851
|
+
*/
|
|
852
|
+
export async function exportSchema(rootDir) {
|
|
853
|
+
const artifact = {};
|
|
854
|
+
const sqliteDir = join(rootDir, 'db', 'sqlite');
|
|
855
|
+
if (existsSync(sqliteDir)) {
|
|
856
|
+
artifact.sqlite = {
|
|
857
|
+
sql: concatMigrations(sqliteDir),
|
|
858
|
+
tables: serializeSchemaMap(await coveredSqliteSchema(sqliteDir)),
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
const postgresDir = join(rootDir, 'db', 'postgres');
|
|
862
|
+
if (existsSync(postgresDir)) {
|
|
863
|
+
artifact.postgres = {
|
|
864
|
+
sql: concatMigrations(postgresDir),
|
|
865
|
+
tables: serializeSchemaMap(await coveredPostgresSchema(postgresDir)),
|
|
866
|
+
};
|
|
867
|
+
}
|
|
868
|
+
return artifact;
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* The schema every wired addon publishes, as sources.
|
|
872
|
+
*
|
|
873
|
+
* An addon never creates its own tables. It has no database of its own — it
|
|
874
|
+
* runs inside the consumer, against the consumer's — so the only honest place
|
|
875
|
+
* for its DDL is the consumer's own migration history, which is what turning it
|
|
876
|
+
* into a `SchemaSource` achieves.
|
|
877
|
+
*/
|
|
878
|
+
export async function addonSchemaSources(rootDir, dialect, addons, logger) {
|
|
879
|
+
if (addons.length === 0)
|
|
880
|
+
return [];
|
|
881
|
+
const require = createRequire(join(rootDir, 'package.json'));
|
|
882
|
+
const sources = [];
|
|
883
|
+
const seen = new Set();
|
|
884
|
+
for (const addon of addons) {
|
|
885
|
+
if (addon.remote || seen.has(addon.package))
|
|
886
|
+
continue;
|
|
887
|
+
seen.add(addon.package);
|
|
888
|
+
let artifactPath;
|
|
889
|
+
try {
|
|
890
|
+
artifactPath = require.resolve(`${addon.package}/${ADDON_DB_ARTIFACT}`);
|
|
891
|
+
}
|
|
892
|
+
catch {
|
|
893
|
+
// Most addons have no schema at all, so an unresolvable artifact is the
|
|
894
|
+
// ordinary case and says nothing is contributed — not that anything failed.
|
|
895
|
+
continue;
|
|
896
|
+
}
|
|
897
|
+
const artifact = JSON.parse(readFileSync(artifactPath, 'utf8'));
|
|
898
|
+
const exported = artifact[dialect];
|
|
899
|
+
if (!exported) {
|
|
900
|
+
logger.error(`The '${addon.package}' addon publishes a schema, but not for ${dialect} — ` +
|
|
901
|
+
`it supports ${Object.keys(artifact).join(', ') || 'no dialect'}. ` +
|
|
902
|
+
'Its tables cannot be created here, so its services will fail at runtime.');
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
sources.push({
|
|
906
|
+
name: addon.package.replace(/^@/, '').replace(/[^a-zA-Z0-9]+/g, '-'),
|
|
907
|
+
desired: {
|
|
908
|
+
tables: deserializeSchemaMap(exported.tables),
|
|
909
|
+
sql: exported.sql,
|
|
910
|
+
},
|
|
911
|
+
origin: `the '${addon.package}' addon`,
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
return sources;
|
|
915
|
+
}
|
|
916
|
+
/**
|
|
917
|
+
* Every schema source the project has, in the order they must be applied.
|
|
918
|
+
*
|
|
919
|
+
* Auth comes first because the runtime's scope tables reference its `user`
|
|
920
|
+
* table; addons come last because they may reference either.
|
|
921
|
+
*/
|
|
922
|
+
export async function schemaSources(resolved, rootDir, srcDirectories, logger, addons = []) {
|
|
923
|
+
const sources = [];
|
|
924
|
+
const auth = await desiredAuthSchema(resolved, rootDir, srcDirectories, logger);
|
|
925
|
+
if (auth) {
|
|
926
|
+
sources.push({
|
|
927
|
+
name: 'better-auth',
|
|
928
|
+
desired: auth,
|
|
929
|
+
origin: 'pikkuBetterAuth (Better Auth)',
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
const runtime = await desiredRuntimeSchema(resolved, rootDir, srcDirectories, logger);
|
|
933
|
+
if (runtime.tables.size > 0) {
|
|
934
|
+
sources.push({
|
|
935
|
+
name: 'pikku-runtime',
|
|
936
|
+
desired: runtime,
|
|
937
|
+
origin: "@pikku/kysely's runtime services",
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
sources.push(...(await addonSchemaSources(rootDir, resolved.dialect, addons, logger)));
|
|
941
|
+
return sources;
|
|
942
|
+
}
|
|
943
|
+
/**
|
|
944
|
+
* Render the columns a table is missing as `ALTER TABLE … ADD COLUMN`.
|
|
945
|
+
*
|
|
946
|
+
* A column that is `NOT NULL` with no default cannot be added to a table that
|
|
947
|
+
* already has rows — the value for those rows is a decision only the author can
|
|
948
|
+
* make. Rather than guessing one or quietly dropping the constraint, the
|
|
949
|
+
* statement is emitted with the problem written above it, so it surfaces during
|
|
950
|
+
* the review the command already asks for.
|
|
951
|
+
*/
|
|
952
|
+
function addColumnStatements(table, columns) {
|
|
953
|
+
const sql = [];
|
|
954
|
+
const needsBackfill = [];
|
|
955
|
+
for (const column of columns) {
|
|
956
|
+
const parts = [
|
|
957
|
+
`ALTER TABLE ${table} ADD COLUMN ${column.name} ${column.type}`,
|
|
958
|
+
];
|
|
959
|
+
if (column.defaultValue !== null) {
|
|
960
|
+
parts.push(`DEFAULT ${column.defaultValue}`);
|
|
961
|
+
}
|
|
962
|
+
if (column.notNull)
|
|
963
|
+
parts.push('NOT NULL');
|
|
964
|
+
if (column.notNull && column.defaultValue === null) {
|
|
965
|
+
needsBackfill.push(`${table}.${column.name}`);
|
|
966
|
+
sql.push(`-- REVIEW: ${column.name} is NOT NULL with no default. This fails if ${table} has rows.\n` +
|
|
967
|
+
`-- Add a default, or split this into: add nullable, backfill, then set NOT NULL.\n` +
|
|
968
|
+
`${parts.join(' ')};`);
|
|
969
|
+
}
|
|
970
|
+
else {
|
|
971
|
+
sql.push(`${parts.join(' ')};`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
return { sql, needsBackfill };
|
|
975
|
+
}
|
|
976
|
+
/**
|
|
977
|
+
* Write a migration for every schema source the migrations do not yet cover.
|
|
978
|
+
*
|
|
979
|
+
* Three cases per source, and the distinction matters. Fully covered is
|
|
980
|
+
* nothing to do. Nothing covered writes the source's own SQL verbatim, which is
|
|
981
|
+
* the one case where the source knows better than any diff — it carries the
|
|
982
|
+
* indexes, constraints and ordering a table-and-column comparison cannot see.
|
|
983
|
+
* Partially covered writes the delta, because re-emitting the whole schema
|
|
984
|
+
* would fail on the tables that already exist.
|
|
985
|
+
*
|
|
986
|
+
* Migrations are written one file per source, numbered in dependency order, so
|
|
987
|
+
* a project can review and apply them independently.
|
|
988
|
+
*/
|
|
989
|
+
export async function generateMigrations(resolved, rootDir, srcDirectories, logger, addons = []) {
|
|
990
|
+
const sources = await schemaSources(resolved, rootDir, srcDirectories, logger, addons);
|
|
991
|
+
const result = { upToDate: [], written: [] };
|
|
992
|
+
for (const source of sources) {
|
|
993
|
+
// Re-read after each write: a migration just written for an earlier source
|
|
994
|
+
// is part of what the next one is compared against.
|
|
995
|
+
const covered = resolved.dialect === 'sqlite'
|
|
996
|
+
? await coveredSqliteSchema(resolved.migrationsDir)
|
|
997
|
+
: await coveredPostgresSchema(resolved.migrationsDir);
|
|
998
|
+
const { missingTables, missingColumns } = diffSchemas(source.desired.tables, covered);
|
|
999
|
+
if (missingTables.length === 0 && missingColumns.length === 0) {
|
|
1000
|
+
result.upToDate.push(source.name);
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
const partial = [...source.desired.tables.keys()].some((t) => covered.has(t));
|
|
1004
|
+
let body;
|
|
1005
|
+
let needsBackfill = [];
|
|
1006
|
+
if (!partial) {
|
|
1007
|
+
body = source.desired.sql;
|
|
1008
|
+
}
|
|
1009
|
+
else {
|
|
1010
|
+
const statements = [];
|
|
1011
|
+
for (const table of missingTables) {
|
|
1012
|
+
const columns = source.desired.tables.get(table);
|
|
1013
|
+
statements.push(`-- REVIEW: ${table} is new. Copy its CREATE TABLE from the source's own SQL —\n` +
|
|
1014
|
+
`-- the column list below carries no indexes, constraints or foreign keys.\n` +
|
|
1015
|
+
`CREATE TABLE ${table} (\n` +
|
|
1016
|
+
[...(columns?.values() ?? [])]
|
|
1017
|
+
.map((c) => ` ${c.name} ${c.type}${c.notNull ? ' NOT NULL' : ''}`)
|
|
1018
|
+
.join(',\n') +
|
|
1019
|
+
'\n);');
|
|
1020
|
+
}
|
|
1021
|
+
for (const { table, columns } of missingColumns) {
|
|
1022
|
+
const infos = columns
|
|
1023
|
+
.map((name) => source.desired.tables.get(table)?.get(name))
|
|
1024
|
+
.filter((c) => c !== undefined);
|
|
1025
|
+
const added = addColumnStatements(table, infos);
|
|
1026
|
+
statements.push(...added.sql);
|
|
1027
|
+
needsBackfill.push(...added.needsBackfill);
|
|
1028
|
+
}
|
|
1029
|
+
body = statements.join('\n\n');
|
|
1030
|
+
}
|
|
1031
|
+
const file = nextMigrationFile(resolved.migrationsDir, source.name);
|
|
1032
|
+
const header = `-- Generated by \`pikku db generate\` from ${source.origin}.\n` +
|
|
1033
|
+
'-- Re-run the command after changing that source.\n\n';
|
|
1034
|
+
writeFileSync(file, header + body + '\n', 'utf8');
|
|
1035
|
+
result.written.push({ source: source.name, file, needsBackfill });
|
|
1036
|
+
}
|
|
1037
|
+
return result;
|
|
653
1038
|
}
|
|
@@ -1,8 +1,20 @@
|
|
|
1
1
|
import { Pool } from 'pg';
|
|
2
|
+
import { MIGRATION_TRACKING_TABLE } from '../db-migrator.js';
|
|
2
3
|
/** Table display name matching `listTables` (schema-qualified unless `public`). */
|
|
3
4
|
function tableKey(schema, table) {
|
|
4
5
|
return schema === 'public' ? table : `${schema}.${table}`;
|
|
5
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* Whether a table is the migrator's own bookkeeping, which no schema owns.
|
|
9
|
+
*
|
|
10
|
+
* Matched on the bare name in any schema: the tracking table is created
|
|
11
|
+
* unqualified, so it lands wherever `search_path` points and there is no one
|
|
12
|
+
* schema to pin it to. The SQLite introspector hides the same name; see
|
|
13
|
+
* {@link MIGRATION_TRACKING_TABLE} for what showing it breaks.
|
|
14
|
+
*/
|
|
15
|
+
function isMigrationTracking(table) {
|
|
16
|
+
return table === MIGRATION_TRACKING_TABLE;
|
|
17
|
+
}
|
|
6
18
|
/**
|
|
7
19
|
* Resolve the SQL type string for a column, preserving array-ness. Postgres
|
|
8
20
|
* reports every array column with `data_type = 'ARRAY'`; the element type lives
|
|
@@ -43,7 +55,9 @@ export class PostgresIntrospector {
|
|
|
43
55
|
AND table_schema NOT LIKE 'pg_temp_%'
|
|
44
56
|
AND table_type = 'BASE TABLE'
|
|
45
57
|
ORDER BY table_schema, table_name`);
|
|
46
|
-
return result.rows
|
|
58
|
+
return result.rows
|
|
59
|
+
.filter((r) => !isMigrationTracking(r.table_name))
|
|
60
|
+
.map((r) => tableKey(r.table_schema, r.table_name));
|
|
47
61
|
}
|
|
48
62
|
async getColumns(table) {
|
|
49
63
|
const dotIdx = table.indexOf('.');
|
|
@@ -141,6 +155,8 @@ export class PostgresIntrospector {
|
|
|
141
155
|
ORDER BY c.table_schema, c.table_name, c.ordinal_position`);
|
|
142
156
|
const byTable = new Map();
|
|
143
157
|
for (const r of result.rows) {
|
|
158
|
+
if (isMigrationTracking(r.table_name))
|
|
159
|
+
continue;
|
|
144
160
|
const key = tableKey(r.table_schema, r.table_name);
|
|
145
161
|
let cols = byTable.get(key);
|
|
146
162
|
if (!cols) {
|
|
@@ -181,6 +197,8 @@ export class PostgresIntrospector {
|
|
|
181
197
|
ORDER BY kcu.table_schema, kcu.table_name, kcu.ordinal_position`);
|
|
182
198
|
const byTable = new Map();
|
|
183
199
|
for (const r of result.rows) {
|
|
200
|
+
if (isMigrationTracking(r.owner_table))
|
|
201
|
+
continue;
|
|
184
202
|
const key = tableKey(r.owner_schema, r.owner_table);
|
|
185
203
|
let fks = byTable.get(key);
|
|
186
204
|
if (!fks) {
|
|
@@ -10,5 +10,6 @@ export declare class PostgresMigrationExecutor implements MigrationExecutor {
|
|
|
10
10
|
constructor(client: PostgresMigrationClient);
|
|
11
11
|
ensureTrackingTable(): Promise<void>;
|
|
12
12
|
getApplied(): Promise<AppliedMigration[]>;
|
|
13
|
+
recordMigration(name: string, hash: string): Promise<void>;
|
|
13
14
|
runMigration(sql: string, name: string, hash: string): Promise<void>;
|
|
14
15
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import { MIGRATION_TRACKING_TABLE as TRACKING_TABLE } from '../db-migrator.js';
|
|
2
2
|
export class PostgresMigrationExecutor {
|
|
3
3
|
client;
|
|
4
4
|
constructor(client) {
|
|
@@ -17,6 +17,9 @@ export class PostgresMigrationExecutor {
|
|
|
17
17
|
const { rows } = await this.client.query(`SELECT name, hash, applied_at FROM ${TRACKING_TABLE} ORDER BY name`);
|
|
18
18
|
return rows;
|
|
19
19
|
}
|
|
20
|
+
async recordMigration(name, hash) {
|
|
21
|
+
await this.client.query(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES ($1, $2)`, [name, hash]);
|
|
22
|
+
}
|
|
20
23
|
async runMigration(sql, name, hash) {
|
|
21
24
|
await this.client.query('BEGIN');
|
|
22
25
|
try {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { MIGRATION_TRACKING_TABLE } from '../db-migrator.js';
|
|
2
|
+
const SKIP_TABLES = new Set(['sqlite_sequence', MIGRATION_TRACKING_TABLE]);
|
|
2
3
|
export class SqliteIntrospector {
|
|
3
4
|
db;
|
|
4
5
|
constructor(db) {
|
|
@@ -5,6 +5,7 @@ export declare class SqliteMigrationExecutor implements MigrationExecutor {
|
|
|
5
5
|
constructor(db: SyncSqliteDatabase);
|
|
6
6
|
ensureTrackingTable(): Promise<void>;
|
|
7
7
|
getApplied(): Promise<AppliedMigration[]>;
|
|
8
|
+
recordMigration(name: string, hash: string): Promise<void>;
|
|
8
9
|
runMigration(sql: string, name: string, hash: string): Promise<void>;
|
|
9
10
|
}
|
|
10
11
|
export declare function dropTrackingTable(db: SyncSqliteDatabase): void;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import { MIGRATION_TRACKING_TABLE as TRACKING_TABLE } from '../db-migrator.js';
|
|
2
2
|
export class SqliteMigrationExecutor {
|
|
3
3
|
db;
|
|
4
4
|
constructor(db) {
|
|
@@ -16,6 +16,11 @@ export class SqliteMigrationExecutor {
|
|
|
16
16
|
.prepare(`SELECT name, hash, applied_at FROM ${TRACKING_TABLE} ORDER BY name`)
|
|
17
17
|
.all();
|
|
18
18
|
}
|
|
19
|
+
async recordMigration(name, hash) {
|
|
20
|
+
this.db
|
|
21
|
+
.prepare(`INSERT INTO ${TRACKING_TABLE} (name, hash) VALUES (?, ?)`)
|
|
22
|
+
.run(name, hash);
|
|
23
|
+
}
|
|
19
24
|
async runMigration(sql, name, hash) {
|
|
20
25
|
this.db.exec('BEGIN');
|
|
21
26
|
try {
|