@databricks-solutions/lakebase-scm-utils 0.2.51 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scripts/index.cjs +2632 -2120
- package/dist/scripts/index.d.cts +1 -1
- package/dist/scripts/index.d.ts +1 -1
- package/dist/scripts/index.js +676 -174
- package/dist/scripts/lakebase/ci-resolve-branch.cli.cjs +2460 -2030
- package/dist/scripts/lakebase/ci-resolve-branch.cli.js +508 -81
- package/dist/scripts/lakebase/collapse-heads.cli.cjs +2429 -2007
- package/dist/scripts/lakebase/collapse-heads.cli.js +477 -58
- package/dist/scripts/lakebase/detect-language.cli.cjs +2441 -2016
- package/dist/scripts/lakebase/detect-language.cli.js +489 -67
- package/dist/scripts/lakebase/doctor.cli.cjs +2484 -2040
- package/dist/scripts/lakebase/doctor.cli.js +532 -91
- package/dist/scripts/lakebase/get-connection.cli.cjs +2 -2
- package/dist/scripts/lakebase/get-connection.cli.js +2 -2
- package/dist/scripts/lakebase/index.cjs +2629 -2117
- package/dist/scripts/lakebase/index.d.cts +59 -10
- package/dist/scripts/lakebase/index.d.ts +59 -10
- package/dist/scripts/lakebase/index.js +670 -168
- package/dist/scripts/lakebase/infra-runner.cli.cjs +2435 -2013
- package/dist/scripts/lakebase/infra-runner.cli.js +483 -64
- package/dist/scripts/lakebase/new-migration.cli.cjs +2429 -2007
- package/dist/scripts/lakebase/new-migration.cli.js +477 -58
- package/dist/scripts/lakebase/schema-diff.cli.cjs +2 -2
- package/dist/scripts/lakebase/schema-diff.cli.js +2 -2
- package/dist/scripts/lakebase/schema-migrate.cli.cjs +2461 -2029
- package/dist/scripts/lakebase/schema-migrate.cli.js +509 -80
- package/dist/scripts/lakebase/scm-claim-feature.cli.cjs +2460 -2030
- package/dist/scripts/lakebase/scm-claim-feature.cli.js +508 -81
- package/dist/scripts/lakebase/scm-doctor.cli.cjs +2486 -2056
- package/dist/scripts/lakebase/scm-doctor.cli.js +537 -107
- package/dist/scripts/lakebase/scm-merge.cli.cjs +2443 -2021
- package/dist/scripts/lakebase/scm-merge.cli.js +494 -72
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.cjs +2464 -2034
- package/dist/scripts/lakebase/scm-reconcile-tier.cli.js +511 -84
- package/dist/scripts/util/index.cjs +2 -2
- package/dist/scripts/util/index.js +2 -2
- package/package.json +1 -1
- package/scripts/lakebase/adapters/efcore-adapter.ts +158 -0
- package/scripts/lakebase/adapters/flyway-adapter.ts +8 -5
- package/scripts/lakebase/collapse-heads.cli.ts +1 -1
- package/scripts/lakebase/create-project.ts +6 -2
- package/scripts/lakebase/detect-language.cli.ts +2 -2
- package/scripts/lakebase/doctor.ts +29 -0
- package/scripts/lakebase/dotnet-project.ts +112 -0
- package/scripts/lakebase/index.ts +1 -0
- package/scripts/lakebase/migration-layout.ts +43 -2
- package/scripts/lakebase/new-migration.cli.ts +1 -1
- package/scripts/lakebase/scaffold-language.ts +23 -2
- package/scripts/lakebase/scaffold.ts +4 -1
- package/scripts/lakebase/schema-migrate-runners/efcore.ts +399 -0
- package/scripts/lakebase/schema-migrate-runners/flyway.ts +6 -4
- package/scripts/lakebase/schema-migrate.cli.ts +3 -1
- package/scripts/lakebase/schema-migrate.ts +24 -10
- package/templates/project/common/.github/workflows/merge.yml +13 -1
- package/templates/project/common/.github/workflows/pr.yml +25 -5
- package/templates/project/common/scripts/flyway-migrate.sh +54 -1
- package/templates/project/common/scripts/prepare-schema-diff.sh +5 -0
- package/templates/project/common/scripts/run-dev.sh +10 -1
- package/templates/project/common/scripts/run-tests.sh +6 -1
- package/templates/project/dotnet/.gitignore.extra +8 -0
- package/templates/project/dotnet/App.sln +56 -0
- package/templates/project/dotnet/Makefile +13 -0
- package/templates/project/dotnet/global.json +6 -0
- package/templates/project/dotnet/src/App/Data/AppDbContext.cs +12 -0
- package/templates/project/dotnet/src/App/Lakebase/DotEnv.cs +48 -0
- package/templates/project/dotnet/src/App/Lakebase/LakebaseConnection.cs +225 -0
- package/templates/project/dotnet/src/App/Program.cs +34 -0
- package/templates/project/dotnet/src/App/appsettings.json +9 -0
- package/templates/project/dotnet/tests/App.Tests/App.Tests.csproj +25 -0
- package/templates/project/dotnet/tests/App.Tests/HealthTests.cs +18 -0
- package/templates/project/dotnet/tests/App.Tests/OpenApiTests.cs +23 -0
- package/templates/project/dotnet-efcore/.config/dotnet-tools.json +13 -0
- package/templates/project/dotnet-efcore/src/App/App.csproj +22 -0
- package/templates/project/dotnet-efcore/src/App/Data/DesignTimeDbContextFactory.cs +24 -0
- package/templates/project/dotnet-efcore/src/App/Migrations/README.md +23 -0
- package/templates/project/dotnet-flyway/db/migrations/README.md +23 -0
- package/templates/project/dotnet-flyway/src/App/App.csproj +17 -0
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
// EF Core runner for .NET projects.
|
|
2
|
+
//
|
|
3
|
+
// Shells to the `dotnet ef` CLI (pinned per project in
|
|
4
|
+
// .config/dotnet-tools.json and restored on first use). Output is read with
|
|
5
|
+
// `--json --prefix-output`, where the JSON payload is the set of lines
|
|
6
|
+
// prefixed `data:`, which gives a stable parse target independent of build
|
|
7
|
+
// and logging noise.
|
|
8
|
+
//
|
|
9
|
+
// Connection: the target branch DSN is passed in the child's DATABASE_URL
|
|
10
|
+
// environment variable, never on the command line, so the credential does
|
|
11
|
+
// not appear in `ps`. The project's IDesignTimeDbContextFactory must read
|
|
12
|
+
// DATABASE_URL (the kit's template does). Projects that cannot can set
|
|
13
|
+
// LAKEBASE_EFCORE_CONNECTION_ARG=1 to pass `--connection` instead (the
|
|
14
|
+
// password is then visible to other local users via `ps`).
|
|
15
|
+
//
|
|
16
|
+
// History lives in "__EFMigrationsHistory" ("MigrationId" = <timestamp>_<Name>).
|
|
17
|
+
// The kit-facing version is the 14-digit timestamp prefix, matching the
|
|
18
|
+
// timestamp versions used for Flyway, Alembic, and Knex.
|
|
19
|
+
|
|
20
|
+
import { spawn } from "node:child_process";
|
|
21
|
+
import * as fs from "node:fs";
|
|
22
|
+
import * as path from "node:path";
|
|
23
|
+
import { Client } from "pg";
|
|
24
|
+
import { EF_DESIGN_PACKAGE, findEfCoreProject, type EfCoreProject } from "../dotnet-project.js";
|
|
25
|
+
import { connectionApplicationName } from "../get-connection.js";
|
|
26
|
+
import {
|
|
27
|
+
SchemaMigrationError,
|
|
28
|
+
type ApplySchemaMigrationsResult,
|
|
29
|
+
type AppliedSchemaMigration,
|
|
30
|
+
type PendingSchemaMigration,
|
|
31
|
+
type RollbackSchemaMigrationResult,
|
|
32
|
+
type SchemaMigrationStatusResult,
|
|
33
|
+
} from "../schema-migrate.js";
|
|
34
|
+
|
|
35
|
+
export interface EfCoreCtx {
|
|
36
|
+
projectDir: string;
|
|
37
|
+
dsn: string;
|
|
38
|
+
/**
|
|
39
|
+
* Reads applied migration ids from "__EFMigrationsHistory" (ascending), or
|
|
40
|
+
* null when the table does not exist. Defaults to a direct query; tests
|
|
41
|
+
* inject a fake.
|
|
42
|
+
*/
|
|
43
|
+
readHistory?: (dsn: string) => Promise<string[] | null>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** One migration as reported by `dotnet ef migrations list --json`. */
|
|
47
|
+
export interface EfMigrationEntry {
|
|
48
|
+
id: string;
|
|
49
|
+
name: string;
|
|
50
|
+
/** true/false with a connection; null with --no-connect. */
|
|
51
|
+
applied: boolean | null;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const MIGRATION_ID = /^(\d{14})_(.+)$/;
|
|
55
|
+
|
|
56
|
+
/** Kit version for an EF migration id: its timestamp prefix. */
|
|
57
|
+
export function efVersion(id: string): string {
|
|
58
|
+
return MIGRATION_ID.exec(id)?.[1] ?? id;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Human description for an EF migration id ("AddMemberPhone" -> "Add Member Phone"). */
|
|
62
|
+
export function efDescription(id: string): string {
|
|
63
|
+
const name = MIGRATION_ID.exec(id)?.[2] ?? id;
|
|
64
|
+
return name.replace(/_/g, " ").replace(/([a-z0-9])([A-Z])/g, "$1 $2");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Convert a postgresql:// DSN into an Npgsql connection string. */
|
|
68
|
+
export function dsnToNpgsql(dsn: string): string {
|
|
69
|
+
const u = new URL(dsn);
|
|
70
|
+
const parts: string[] = [
|
|
71
|
+
`Host=${u.hostname}`,
|
|
72
|
+
`Port=${u.port || "5432"}`,
|
|
73
|
+
`Database=${decodeURIComponent(u.pathname.replace(/^\//, "")) || "databricks_postgres"}`,
|
|
74
|
+
];
|
|
75
|
+
if (u.username) parts.push(`Username=${decodeURIComponent(u.username)}`);
|
|
76
|
+
if (u.password) parts.push(`Password=${decodeURIComponent(u.password)}`);
|
|
77
|
+
const sslmode = (u.searchParams.get("sslmode") ?? "require").toLowerCase();
|
|
78
|
+
const ssl: Record<string, string> = {
|
|
79
|
+
disable: "Disable",
|
|
80
|
+
allow: "Allow",
|
|
81
|
+
prefer: "Prefer",
|
|
82
|
+
require: "Require",
|
|
83
|
+
"verify-ca": "VerifyCA",
|
|
84
|
+
"verify-full": "VerifyFull",
|
|
85
|
+
};
|
|
86
|
+
parts.push(`SSL Mode=${ssl[sslmode] ?? "Require"}`);
|
|
87
|
+
const app = u.searchParams.get("application_name");
|
|
88
|
+
if (app) parts.push(`Application Name=${app}`);
|
|
89
|
+
return parts.map((p) => p.replace(/;/g, "")).join(";");
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Extract the JSON payload from `--prefix-output` output (lines starting `data:`). */
|
|
93
|
+
export function parseEfJson<T>(stdout: string): T {
|
|
94
|
+
const body = stdout
|
|
95
|
+
.split(/\r?\n/)
|
|
96
|
+
.filter((l) => l.startsWith("data:"))
|
|
97
|
+
.map((l) => l.slice("data:".length))
|
|
98
|
+
.join("\n")
|
|
99
|
+
.trim();
|
|
100
|
+
if (!body) {
|
|
101
|
+
throw new SchemaMigrationError(`dotnet ef returned no JSON payload: ${stdout.slice(0, 400)}`);
|
|
102
|
+
}
|
|
103
|
+
try {
|
|
104
|
+
return JSON.parse(body) as T;
|
|
105
|
+
} catch (err) {
|
|
106
|
+
throw new SchemaMigrationError(
|
|
107
|
+
`dotnet ef JSON parse failed: ${err instanceof Error ? err.message : String(err)}.\n` +
|
|
108
|
+
`Body (first 400 chars): ${body.slice(0, 400)}`
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function requireProject(projectDir: string): EfCoreProject {
|
|
114
|
+
const project = findEfCoreProject(projectDir);
|
|
115
|
+
if (!project) {
|
|
116
|
+
throw new SchemaMigrationError(
|
|
117
|
+
`No EF Core project found under ${projectDir}. Expected a *.csproj that references ${EF_DESIGN_PACKAGE}.`
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
return project;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function spawnDotnet(
|
|
124
|
+
projectDir: string,
|
|
125
|
+
args: string[],
|
|
126
|
+
env: NodeJS.ProcessEnv
|
|
127
|
+
): Promise<{ stdout: string; stderr: string }> {
|
|
128
|
+
return new Promise((resolve, reject) => {
|
|
129
|
+
const child = spawn("dotnet", args, {
|
|
130
|
+
cwd: projectDir,
|
|
131
|
+
env: { ...process.env, DOTNET_NOLOGO: "1", DOTNET_CLI_TELEMETRY_OPTOUT: "1", ...env },
|
|
132
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
133
|
+
});
|
|
134
|
+
let stdout = "";
|
|
135
|
+
let stderr = "";
|
|
136
|
+
child.stdout.on("data", (chunk: Buffer) => {
|
|
137
|
+
stdout += chunk.toString("utf8");
|
|
138
|
+
});
|
|
139
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
140
|
+
stderr += chunk.toString("utf8");
|
|
141
|
+
});
|
|
142
|
+
child.on("error", (err) => {
|
|
143
|
+
reject(
|
|
144
|
+
new SchemaMigrationError(
|
|
145
|
+
`Could not spawn dotnet. Is the .NET SDK installed and on PATH? ${err.message}`,
|
|
146
|
+
err
|
|
147
|
+
)
|
|
148
|
+
);
|
|
149
|
+
});
|
|
150
|
+
child.on("close", (code) => {
|
|
151
|
+
if (code === 0) {
|
|
152
|
+
resolve({ stdout, stderr });
|
|
153
|
+
} else {
|
|
154
|
+
reject(
|
|
155
|
+
new SchemaMigrationError(
|
|
156
|
+
`dotnet ${args.join(" ")} exited with code ${code}.\nstdout: ${stdout}\nstderr: ${stderr}`
|
|
157
|
+
)
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const restored = new Set<string>();
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Once per project per process: `dotnet tool restore` (when a tool manifest
|
|
168
|
+
* exists) and `dotnet restore` of the EF project. `dotnet ef` reads project
|
|
169
|
+
* metadata before it builds, which fails on a fresh clone without restored
|
|
170
|
+
* assets.
|
|
171
|
+
*/
|
|
172
|
+
async function ensureRestored(projectDir: string, csproj: string): Promise<void> {
|
|
173
|
+
if (restored.has(projectDir)) return;
|
|
174
|
+
if (fs.existsSync(path.join(projectDir, ".config", "dotnet-tools.json"))) {
|
|
175
|
+
await spawnDotnet(projectDir, ["tool", "restore"], {});
|
|
176
|
+
}
|
|
177
|
+
await spawnDotnet(projectDir, ["restore", csproj], {});
|
|
178
|
+
restored.add(projectDir);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Run `dotnet ef <args>` against the project's EF Core project. `noBuild`
|
|
183
|
+
* skips the implicit build, for follow-up calls in the same operation.
|
|
184
|
+
*/
|
|
185
|
+
async function runEf(
|
|
186
|
+
projectDir: string,
|
|
187
|
+
args: string[],
|
|
188
|
+
dsn?: string,
|
|
189
|
+
opts: { noBuild?: boolean } = {}
|
|
190
|
+
): Promise<{ stdout: string; stderr: string }> {
|
|
191
|
+
const project = requireProject(projectDir);
|
|
192
|
+
const csproj = path.relative(projectDir, project.csproj) || project.csproj;
|
|
193
|
+
await ensureRestored(projectDir, csproj);
|
|
194
|
+
const full = ["ef", ...args, "--project", csproj, "--prefix-output"];
|
|
195
|
+
if (opts.noBuild) full.push("--no-build");
|
|
196
|
+
const env: NodeJS.ProcessEnv = {};
|
|
197
|
+
if (dsn) {
|
|
198
|
+
env.DATABASE_URL = dsn;
|
|
199
|
+
if (process.env.LAKEBASE_EFCORE_CONNECTION_ARG === "1") full.push("--connection", dsnToNpgsql(dsn));
|
|
200
|
+
}
|
|
201
|
+
return spawnDotnet(projectDir, full, env);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* List migrations with their applied state on the target database. EF reports
|
|
206
|
+
* `applied: null` when it cannot reach the database (exit code 0), so that is
|
|
207
|
+
* an error here rather than "nothing applied".
|
|
208
|
+
*/
|
|
209
|
+
export async function listEfMigrations(
|
|
210
|
+
ctx: EfCoreCtx,
|
|
211
|
+
opts: { noBuild?: boolean } = {}
|
|
212
|
+
): Promise<EfMigrationEntry[]> {
|
|
213
|
+
const { stdout, stderr } = await runEf(ctx.projectDir, ["migrations", "list", "--json"], ctx.dsn, opts);
|
|
214
|
+
const entries = parseEfJson<EfMigrationEntry[]>(stdout);
|
|
215
|
+
if (ctx.dsn && entries.some((e) => e.applied === null)) {
|
|
216
|
+
const warning = stdout
|
|
217
|
+
.split(/\r?\n/)
|
|
218
|
+
.filter((l) => /^(warn|error):/.test(l))
|
|
219
|
+
.join("\n");
|
|
220
|
+
throw new SchemaMigrationError(
|
|
221
|
+
`dotnet ef could not read migration state from the database (check connectivity and credentials).\n${warning || stderr}`.trim()
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
return entries;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** Applied migration ids from "__EFMigrationsHistory", ascending; null when the table is absent. */
|
|
228
|
+
export async function readEfHistory(dsn: string): Promise<string[] | null> {
|
|
229
|
+
const client = new Client({ connectionString: dsn, application_name: connectionApplicationName() });
|
|
230
|
+
await client.connect();
|
|
231
|
+
try {
|
|
232
|
+
const exists = await client.query(`SELECT to_regclass('"__EFMigrationsHistory"') IS NOT NULL AS present`);
|
|
233
|
+
if (!exists.rows[0]?.present) return null;
|
|
234
|
+
const r = await client.query(`SELECT "MigrationId" FROM "__EFMigrationsHistory" ORDER BY "MigrationId"`);
|
|
235
|
+
return r.rows.map((row: { MigrationId: string }) => row.MigrationId);
|
|
236
|
+
} finally {
|
|
237
|
+
await client.end();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function toApplied(entries: EfMigrationEntry[]): AppliedSchemaMigration[] {
|
|
242
|
+
return entries.map((e) => ({ version: efVersion(e.id), description: efDescription(e.id) }));
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* `current` comes from "__EFMigrationsHistory" itself, not from
|
|
247
|
+
* `migrations list` (which only knows migrations compiled into the project),
|
|
248
|
+
* so a database that is ahead of the code reports its real head and the
|
|
249
|
+
* kit's orphan check can see it.
|
|
250
|
+
*/
|
|
251
|
+
export async function statusEfcore(ctx: EfCoreCtx): Promise<SchemaMigrationStatusResult> {
|
|
252
|
+
const entries = await listEfMigrations(ctx);
|
|
253
|
+
const history = await (ctx.readHistory ?? readEfHistory)(ctx.dsn);
|
|
254
|
+
const current = history && history.length > 0 ? efVersion(history[history.length - 1]) : undefined;
|
|
255
|
+
const pending: PendingSchemaMigration[] = entries
|
|
256
|
+
.filter((e) => e.applied !== true)
|
|
257
|
+
.map((e) => ({ version: efVersion(e.id), filename: `${e.id}.cs`, description: efDescription(e.id) }));
|
|
258
|
+
return { current, pending, tool: "efcore" };
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export async function applyEfcore(ctx: EfCoreCtx): Promise<ApplySchemaMigrationsResult> {
|
|
262
|
+
const before = new Set((await listEfMigrations(ctx)).filter((e) => e.applied).map((e) => e.id));
|
|
263
|
+
await runEf(ctx.projectDir, ["database", "update"], ctx.dsn, { noBuild: true });
|
|
264
|
+
const after = await listEfMigrations(ctx, { noBuild: true });
|
|
265
|
+
const applied = toApplied(after.filter((e) => e.applied && !before.has(e.id)));
|
|
266
|
+
return { applied, alreadyAtLatest: applied.length === 0, tool: "efcore" };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* Resolve a rollback target to an EF migration id (or "0" for an empty
|
|
271
|
+
* database). Accepts "0", a relative step ("-1", "-2"), a full migration id,
|
|
272
|
+
* its timestamp version, or its name.
|
|
273
|
+
*/
|
|
274
|
+
export function resolveEfTarget(target: string, applied: EfMigrationEntry[]): string {
|
|
275
|
+
const t = target.trim();
|
|
276
|
+
if (t === "0") return "0";
|
|
277
|
+
const rel = /^-(\d+)$/.exec(t);
|
|
278
|
+
if (rel) {
|
|
279
|
+
const steps = Number(rel[1]);
|
|
280
|
+
if (steps < 1) throw new SchemaMigrationError(`Invalid rollback step: ${target}`);
|
|
281
|
+
const index = applied.length - 1 - steps;
|
|
282
|
+
if (index < -1) {
|
|
283
|
+
throw new SchemaMigrationError(`Cannot roll back ${steps} migration(s); only ${applied.length} applied.`);
|
|
284
|
+
}
|
|
285
|
+
return index === -1 ? "0" : applied[index].id;
|
|
286
|
+
}
|
|
287
|
+
const match = applied.find((e) => e.id === t || efVersion(e.id) === t || e.name === t);
|
|
288
|
+
if (!match) throw new SchemaMigrationError(`Rollback target '${target}' is not an applied migration.`);
|
|
289
|
+
return match.id;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
export async function rollbackEfcore(ctx: EfCoreCtx & { target: string }): Promise<RollbackSchemaMigrationResult> {
|
|
293
|
+
const beforeEntries = (await listEfMigrations(ctx)).filter((e) => e.applied);
|
|
294
|
+
const targetId = resolveEfTarget(ctx.target, beforeEntries);
|
|
295
|
+
await runEf(ctx.projectDir, ["database", "update", targetId], ctx.dsn, { noBuild: true });
|
|
296
|
+
const stillApplied = new Set((await listEfMigrations(ctx, { noBuild: true })).filter((e) => e.applied).map((e) => e.id));
|
|
297
|
+
return { rolledBack: toApplied(beforeEntries.filter((e) => !stillApplied.has(e.id)).reverse()), tool: "efcore" };
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** C# identifier for `migrations add` ("add member phone" -> "AddMemberPhone"). */
|
|
301
|
+
export function efMigrationName(slug: string): string {
|
|
302
|
+
const name = slug
|
|
303
|
+
.split(/[^A-Za-z0-9]+/)
|
|
304
|
+
.filter(Boolean)
|
|
305
|
+
.map((w) => w[0].toUpperCase() + w.slice(1))
|
|
306
|
+
.join("");
|
|
307
|
+
if (!name) throw new SchemaMigrationError(`Cannot derive a migration name from '${slug}'.`);
|
|
308
|
+
return /^[0-9]/.test(name) ? `M${name}` : name;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* `dotnet ef migrations add <Name>`: EF generates the migration from the
|
|
313
|
+
* model diff (the EF equivalent of Alembic autogenerate). No DB needed.
|
|
314
|
+
*/
|
|
315
|
+
export async function createEfcoreMigration(opts: {
|
|
316
|
+
projectDir: string;
|
|
317
|
+
slug: string;
|
|
318
|
+
}): Promise<{ id: string; path: string }> {
|
|
319
|
+
const project = requireProject(opts.projectDir);
|
|
320
|
+
const name = efMigrationName(opts.slug);
|
|
321
|
+
const existing = new Set(fs.existsSync(project.migrationsDir) ? fs.readdirSync(project.migrationsDir) : []);
|
|
322
|
+
await runEf(opts.projectDir, ["migrations", "add", name]);
|
|
323
|
+
const created = fs
|
|
324
|
+
.readdirSync(project.migrationsDir)
|
|
325
|
+
.filter((f) => !existing.has(f) && MIGRATION_ID.test(f.replace(/\.cs$/, "")) && !f.endsWith(".Designer.cs"));
|
|
326
|
+
if (created.length !== 1) {
|
|
327
|
+
throw new SchemaMigrationError(`Expected one new migration file for ${name}, found: ${created.join(", ") || "none"}`);
|
|
328
|
+
}
|
|
329
|
+
return { id: created[0].replace(/\.cs$/, ""), path: path.join(project.migrationsDir, created[0]) };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Local EF migration ids in apply order (from Migrations/*.cs, no build). */
|
|
333
|
+
export function localEfMigrationIds(projectDir: string): string[] {
|
|
334
|
+
const project = findEfCoreProject(projectDir);
|
|
335
|
+
if (!project || !fs.existsSync(project.migrationsDir)) return [];
|
|
336
|
+
return fs
|
|
337
|
+
.readdirSync(project.migrationsDir)
|
|
338
|
+
.filter((f) => f.endsWith(".cs") && !f.endsWith(".Designer.cs") && MIGRATION_ID.test(f.slice(0, -3)))
|
|
339
|
+
.map((f) => f.slice(0, -3))
|
|
340
|
+
.sort();
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** EF Core version recorded in "ProductVersion" when stamping. */
|
|
344
|
+
function efProductVersion(project: EfCoreProject): string {
|
|
345
|
+
try {
|
|
346
|
+
const csproj = fs.readFileSync(project.csproj, "utf-8");
|
|
347
|
+
const m = new RegExp(`${EF_DESIGN_PACKAGE.replace(/\./g, "\\.")}"\\s+Version="([^"]+)"`).exec(csproj);
|
|
348
|
+
if (m) return m[1];
|
|
349
|
+
} catch {
|
|
350
|
+
/* fall through */
|
|
351
|
+
}
|
|
352
|
+
return "unknown";
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* Set "__EFMigrationsHistory" to exactly the local migrations up to
|
|
357
|
+
* `revision` ("head" = all local) WITHOUT running them: rows for those ids are
|
|
358
|
+
* added and every other row (newer migrations, or orphans whose files are
|
|
359
|
+
* gone) is removed, matching Alembic's stamp. Used to adopt a database whose
|
|
360
|
+
* schema already matches, and by reconcile for a database ahead of code. EF
|
|
361
|
+
* has no native stamp command, so this writes the table in one transaction.
|
|
362
|
+
*/
|
|
363
|
+
export async function stampEfcore(ctx: EfCoreCtx & { revision: string }): Promise<string> {
|
|
364
|
+
const project = requireProject(ctx.projectDir);
|
|
365
|
+
const ids = localEfMigrationIds(ctx.projectDir);
|
|
366
|
+
if (ids.length === 0) throw new SchemaMigrationError("No local EF Core migrations to stamp.");
|
|
367
|
+
const rev = ctx.revision.trim();
|
|
368
|
+
const end = rev === "head" ? ids.length - 1 : ids.findIndex((id) => id === rev || efVersion(id) === rev);
|
|
369
|
+
if (end < 0) throw new SchemaMigrationError(`Stamp revision '${ctx.revision}' is not a local migration.`);
|
|
370
|
+
const toStamp = ids.slice(0, end + 1);
|
|
371
|
+
const productVersion = efProductVersion(project);
|
|
372
|
+
|
|
373
|
+
const client = new Client({ connectionString: ctx.dsn, application_name: connectionApplicationName() });
|
|
374
|
+
await client.connect();
|
|
375
|
+
try {
|
|
376
|
+
await client.query("BEGIN");
|
|
377
|
+
await client.query(
|
|
378
|
+
`CREATE TABLE IF NOT EXISTS "__EFMigrationsHistory" (
|
|
379
|
+
"MigrationId" character varying(150) NOT NULL,
|
|
380
|
+
"ProductVersion" character varying(32) NOT NULL,
|
|
381
|
+
CONSTRAINT "PK___EFMigrationsHistory" PRIMARY KEY ("MigrationId"))`
|
|
382
|
+
);
|
|
383
|
+
await client.query(`DELETE FROM "__EFMigrationsHistory" WHERE NOT ("MigrationId" = ANY($1::text[]))`, [toStamp]);
|
|
384
|
+
for (const id of toStamp) {
|
|
385
|
+
await client.query(
|
|
386
|
+
`INSERT INTO "__EFMigrationsHistory" ("MigrationId", "ProductVersion") VALUES ($1, $2)
|
|
387
|
+
ON CONFLICT ("MigrationId") DO NOTHING`,
|
|
388
|
+
[id, productVersion]
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
await client.query("COMMIT");
|
|
392
|
+
} catch (err) {
|
|
393
|
+
await client.query("ROLLBACK").catch(() => undefined);
|
|
394
|
+
throw err;
|
|
395
|
+
} finally {
|
|
396
|
+
await client.end();
|
|
397
|
+
}
|
|
398
|
+
return efVersion(toStamp[toStamp.length - 1]);
|
|
399
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
// Flyway runner for Java + Kotlin projects
|
|
2
|
-
// primitives lift).
|
|
1
|
+
// Flyway runner for Java + Kotlin projects, and .NET projects that use
|
|
2
|
+
// Flyway (part of the primitives lift).
|
|
3
3
|
//
|
|
4
4
|
// Shells to the standalone Flyway Community Edition CLI in JSON output
|
|
5
5
|
// mode. The runner reads the project's migration files from
|
|
6
|
-
// `src/main/resources/db/migration/` (Flyway's standard layout)
|
|
6
|
+
// `src/main/resources/db/migration/` (Flyway's standard Maven layout) or,
|
|
7
|
+
// for .NET, `db/migrations/` (see flywayMigrationsDir). It
|
|
7
8
|
// passes connection info through FLYWAY_URL / FLYWAY_USER /
|
|
8
9
|
// FLYWAY_PASSWORD env vars rather than command-line flags so the
|
|
9
10
|
// secret never appears in `ps`.
|
|
@@ -25,6 +26,7 @@
|
|
|
25
26
|
|
|
26
27
|
import { spawn } from "node:child_process";
|
|
27
28
|
import * as path from "node:path";
|
|
29
|
+
import { flywayMigrationsDir } from "../dotnet-project.js";
|
|
28
30
|
import {
|
|
29
31
|
SchemaMigrationError,
|
|
30
32
|
type ApplySchemaMigrationsResult,
|
|
@@ -52,7 +54,7 @@ function dsnToFlywayEnv(dsn: string): { url: string; user: string; password: str
|
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
function migrationsLocation(projectDir: string): string {
|
|
55
|
-
return `filesystem:${
|
|
57
|
+
return `filesystem:${flywayMigrationsDir(projectDir)}`;
|
|
56
58
|
}
|
|
57
59
|
|
|
58
60
|
interface FlywayRun {
|
|
@@ -94,7 +94,7 @@ Common flags (for apply, rollback, status):
|
|
|
94
94
|
--instance <id> Lakebase project id (required)
|
|
95
95
|
--branch <name> Branch to migrate against (required)
|
|
96
96
|
--project-dir <dir> Project root (default: cwd)
|
|
97
|
-
--language <lang> java | kotlin | python | nodejs (default: auto-detect)
|
|
97
|
+
--language <lang> java | kotlin | python | nodejs | dotnet (default: auto-detect)
|
|
98
98
|
--database <db> Database name (default: $PGDATABASE or "databricks_postgres")
|
|
99
99
|
--endpoint <name> Endpoint identifier on the branch (default: "primary")
|
|
100
100
|
|
|
@@ -115,6 +115,8 @@ Language support today:
|
|
|
115
115
|
python (alembic) Full: apply, rollback, status, list
|
|
116
116
|
java, kotlin Full: apply, status, list (rollback unsupported in Flyway Community Edition)
|
|
117
117
|
nodejs (knex) Full: apply, rollback, status, list
|
|
118
|
+
dotnet (efcore) Full: apply, rollback, status, list (stamp via the adapter)
|
|
119
|
+
dotnet (flyway) Full: apply, status, list (db/migrations/; rollback unsupported)
|
|
118
120
|
`;
|
|
119
121
|
}
|
|
120
122
|
|
|
@@ -30,6 +30,9 @@ import { normalizeTierName, protectedTierNamesFromEnv } from "./branch-utils.js"
|
|
|
30
30
|
import "./adapters/alembic-adapter.js";
|
|
31
31
|
import "./adapters/flyway-adapter.js";
|
|
32
32
|
import "./adapters/knex-adapter.js";
|
|
33
|
+
import "./adapters/efcore-adapter.js";
|
|
34
|
+
import { isDotnetFlywayProject, flywayMigrationsDir } from "./dotnet-project.js";
|
|
35
|
+
import { listEfCoreFiles } from "./adapters/efcore-adapter.js";
|
|
33
36
|
import {
|
|
34
37
|
resolveSchemaMigrationAdapter,
|
|
35
38
|
type SchemaMigrationAdapterId,
|
|
@@ -38,9 +41,9 @@ import {
|
|
|
38
41
|
type CollapseHeadsResult,
|
|
39
42
|
} from "./schema-migration-adapter.js";
|
|
40
43
|
|
|
41
|
-
export type SchemaMigrationLanguage = "java" | "kotlin" | "python" | "nodejs";
|
|
44
|
+
export type SchemaMigrationLanguage = "java" | "kotlin" | "python" | "nodejs" | "dotnet";
|
|
42
45
|
|
|
43
|
-
export type SchemaMigrationToolName = "flyway" | "alembic" | "knex";
|
|
46
|
+
export type SchemaMigrationToolName = "flyway" | "alembic" | "knex" | "efcore";
|
|
44
47
|
|
|
45
48
|
export interface SchemaMigrationFile {
|
|
46
49
|
/** Stable identifier sortable in apply-order: Flyway `V<n>`, Alembic
|
|
@@ -48,7 +51,7 @@ export interface SchemaMigrationFile {
|
|
|
48
51
|
version: string;
|
|
49
52
|
filename: string;
|
|
50
53
|
description: string;
|
|
51
|
-
type: "SQL" | "Python" | "JavaScript" | "TypeScript";
|
|
54
|
+
type: "SQL" | "Python" | "JavaScript" | "TypeScript" | "CSharp";
|
|
52
55
|
/** Tool that should run this file. */
|
|
53
56
|
tool: SchemaMigrationToolName;
|
|
54
57
|
}
|
|
@@ -149,16 +152,24 @@ export function detectLanguage(projectDir: string): SchemaMigrationLanguage {
|
|
|
149
152
|
if (lang === "unknown") {
|
|
150
153
|
throw new SchemaMigrationError(
|
|
151
154
|
`Could not detect project language in ${projectDir}. ` +
|
|
152
|
-
`Expected one of: pom.xml (java/kotlin), pyproject.toml or alembic.ini (python), package.json (nodejs). ` +
|
|
155
|
+
`Expected one of: pom.xml (java/kotlin), pyproject.toml or alembic.ini (python), package.json (nodejs), *.sln or *.csproj (dotnet). ` +
|
|
153
156
|
`Pass {language} explicitly to override.`
|
|
154
157
|
);
|
|
155
158
|
}
|
|
156
159
|
return lang;
|
|
157
160
|
}
|
|
158
161
|
|
|
159
|
-
/**
|
|
160
|
-
|
|
162
|
+
/**
|
|
163
|
+
* Map a language to the migration tool the kit invokes for it. .NET supports
|
|
164
|
+
* two tools, so it also needs the project directory: EF Core when a project
|
|
165
|
+
* references Microsoft.EntityFrameworkCore.Design, Flyway when SQL migrations
|
|
166
|
+
* live in db/migrations/ (Flyway wins when both are present), else EF Core.
|
|
167
|
+
*/
|
|
168
|
+
export function toolForLanguage(language: SchemaMigrationLanguage, projectDir?: string): SchemaMigrationToolName {
|
|
161
169
|
switch (language) {
|
|
170
|
+
case "dotnet":
|
|
171
|
+
if (projectDir && isDotnetFlywayProject(projectDir)) return "flyway";
|
|
172
|
+
return "efcore";
|
|
162
173
|
case "java":
|
|
163
174
|
case "kotlin":
|
|
164
175
|
return "flyway";
|
|
@@ -182,9 +193,11 @@ export function toolForLanguage(language: SchemaMigrationLanguage): SchemaMigrat
|
|
|
182
193
|
export function listSchemaMigrations(args: ListSchemaMigrationsArgs = {}): SchemaMigrationFile[] {
|
|
183
194
|
const projectDir = args.projectDir ?? process.cwd();
|
|
184
195
|
const language = args.language ?? detectLanguage(projectDir);
|
|
185
|
-
const tool = toolForLanguage(language);
|
|
196
|
+
const tool = toolForLanguage(language, projectDir);
|
|
186
197
|
|
|
187
198
|
switch (tool) {
|
|
199
|
+
case "efcore":
|
|
200
|
+
return listEfCoreFiles(projectDir);
|
|
188
201
|
case "flyway":
|
|
189
202
|
return listFlywayMigrations(projectDir);
|
|
190
203
|
case "alembic":
|
|
@@ -195,8 +208,9 @@ export function listSchemaMigrations(args: ListSchemaMigrationsArgs = {}): Schem
|
|
|
195
208
|
}
|
|
196
209
|
|
|
197
210
|
function listFlywayMigrations(projectDir: string): SchemaMigrationFile[] {
|
|
198
|
-
// Flyway convention:
|
|
199
|
-
|
|
211
|
+
// Flyway convention: V<n>__<desc>.sql under the Maven layout (Java/Kotlin)
|
|
212
|
+
// or db/migrations/ (.NET); see flywayMigrationsDir.
|
|
213
|
+
const dir = flywayMigrationsDir(projectDir);
|
|
200
214
|
if (!fs.existsSync(dir)) return [];
|
|
201
215
|
const files = fs.readdirSync(dir).filter((f) => /^V\d+(\.\d+)*__.+\.sql$/.test(f));
|
|
202
216
|
return files
|
|
@@ -291,7 +305,7 @@ async function dsnFor(args: {
|
|
|
291
305
|
/** Resolve an adapter, honoring an explicit language override. */
|
|
292
306
|
function adapterFor(projectDir: string, language?: SchemaMigrationLanguage) {
|
|
293
307
|
const override: SchemaMigrationAdapterId | undefined = language
|
|
294
|
-
? toolForLanguage(language)
|
|
308
|
+
? toolForLanguage(language, projectDir)
|
|
295
309
|
: undefined;
|
|
296
310
|
return resolveSchemaMigrationAdapter(projectDir, override);
|
|
297
311
|
}
|
|
@@ -143,6 +143,18 @@ jobs:
|
|
|
143
143
|
fi
|
|
144
144
|
npm install
|
|
145
145
|
|
|
146
|
+
- name: Set up .NET
|
|
147
|
+
if: steps.detect-lang.outputs.lang == 'dotnet'
|
|
148
|
+
uses: actions/setup-dotnet@v4
|
|
149
|
+
with:
|
|
150
|
+
global-json-file: global.json
|
|
151
|
+
|
|
152
|
+
- name: Restore .NET packages and tools
|
|
153
|
+
if: steps.detect-lang.outputs.lang == 'dotnet'
|
|
154
|
+
run: |
|
|
155
|
+
dotnet restore
|
|
156
|
+
if [ -f .config/dotnet-tools.json ]; then dotnet tool restore; fi
|
|
157
|
+
|
|
146
158
|
# The official action installs to a user-writable path (no sudo). It's
|
|
147
159
|
# a no-op if a compatible databricks CLI is already on PATH.
|
|
148
160
|
- name: Install Databricks CLI
|
|
@@ -218,7 +230,7 @@ jobs:
|
|
|
218
230
|
# 3. Default https://repo1.maven.org/maven2 for GitHub-hosted
|
|
219
231
|
# runners with public internet.
|
|
220
232
|
- name: Install Flyway CLI
|
|
221
|
-
if: hashFiles('pom.xml') != ''
|
|
233
|
+
if: (hashFiles('pom.xml') != '' || hashFiles('db/migrations/**') != '')
|
|
222
234
|
env:
|
|
223
235
|
FLYWAY_DOWNLOAD_BASE_URL: ${{ vars.FLYWAY_DOWNLOAD_BASE_URL }}
|
|
224
236
|
run: |
|
|
@@ -102,6 +102,18 @@ jobs:
|
|
|
102
102
|
fi
|
|
103
103
|
npm install
|
|
104
104
|
|
|
105
|
+
- name: Set up .NET
|
|
106
|
+
if: steps.detect-lang.outputs.lang == 'dotnet'
|
|
107
|
+
uses: actions/setup-dotnet@v4
|
|
108
|
+
with:
|
|
109
|
+
global-json-file: global.json
|
|
110
|
+
|
|
111
|
+
- name: Restore .NET packages and tools
|
|
112
|
+
if: steps.detect-lang.outputs.lang == 'dotnet'
|
|
113
|
+
run: |
|
|
114
|
+
dotnet restore
|
|
115
|
+
if [ -f .config/dotnet-tools.json ]; then dotnet tool restore; fi
|
|
116
|
+
|
|
105
117
|
# Install frontend dependencies when a `client/` workspace is present
|
|
106
118
|
# (Python+React layouts etc.). Gated on client/package.json so projects
|
|
107
119
|
# without a frontend skip this transparently.
|
|
@@ -301,10 +313,12 @@ jobs:
|
|
|
301
313
|
BODY="⚠️ **No Lakebase CI branch** was created. This run is not using a dedicated database branch. Open the [workflow run](${RUN_URL}), expand **Create CI database branch (Lakebase)** and check the step **Summary** (or log) for the reason and how to fix it."
|
|
302
314
|
gh pr comment "$PR" --body "$BODY" 2>/dev/null || true
|
|
303
315
|
|
|
304
|
-
# Install Flyway CLI for Java/Kotlin projects
|
|
305
|
-
# migrate runner spawns the
|
|
306
|
-
# projects don't need it. Runtime-gated
|
|
307
|
-
#
|
|
316
|
+
# Install Flyway CLI for Java/Kotlin projects and .NET projects that use
|
|
317
|
+
# Flyway (db/migrations/). The substrate's migrate runner spawns the
|
|
318
|
+
# standalone `flyway` binary; other projects don't need it. Runtime-gated
|
|
319
|
+
# on pom.xml / any file under db/migrations/ (what the adapter checks; a
|
|
320
|
+
# fresh scaffold has only its README there) so the YAML is one shape for
|
|
321
|
+
# every language.
|
|
308
322
|
#
|
|
309
323
|
# Resolution order:
|
|
310
324
|
# 1. `flyway` already on PATH (self-hosted runner with brew /
|
|
@@ -315,7 +329,7 @@ jobs:
|
|
|
315
329
|
# 3. Default https://repo1.maven.org/maven2 for GitHub-hosted
|
|
316
330
|
# runners with public internet.
|
|
317
331
|
- name: Install Flyway CLI
|
|
318
|
-
if: steps.lakebase-db.outputs.url != '' && hashFiles('pom.xml') != ''
|
|
332
|
+
if: steps.lakebase-db.outputs.url != '' && (hashFiles('pom.xml') != '' || hashFiles('db/migrations/**') != '')
|
|
319
333
|
env:
|
|
320
334
|
FLYWAY_DOWNLOAD_BASE_URL: ${{ vars.FLYWAY_DOWNLOAD_BASE_URL }}
|
|
321
335
|
run: |
|
|
@@ -388,6 +402,8 @@ jobs:
|
|
|
388
402
|
uv run pytest
|
|
389
403
|
elif [ "$LANG" = "nodejs" ]; then
|
|
390
404
|
npm test
|
|
405
|
+
elif [ "$LANG" = "dotnet" ]; then
|
|
406
|
+
dotnet test --no-restore
|
|
391
407
|
fi
|
|
392
408
|
else
|
|
393
409
|
echo "::error::No CI branch URL – cannot run tests without a database. Check the 'Create CI database branch' step for details."
|
|
@@ -570,6 +586,10 @@ jobs:
|
|
|
570
586
|
desc=$(echo "$base" | sed 's/^[^_]*_//' | sed 's/\.py$//' | tr '_' ' ')
|
|
571
587
|
[ -n "$rev" ] && echo "| $rev | $desc |" >> "$MD"
|
|
572
588
|
done
|
|
589
|
+
elif [ "$LANG" = "dotnet" ]; then
|
|
590
|
+
# EF Core or Flyway: the kit's canonical discovery picks the tool + files.
|
|
591
|
+
npx --yes --package="${SCM_UTILS_NPX_PKG}" lakebase-schema-migrate list \
|
|
592
|
+
| jq -r '.[] | if .tool == "flyway" then "| V\(.version) | \(.description) |" else "| \(.version) | \(.description) (EF Core) |" end' >> "$MD"
|
|
573
593
|
elif [ "$LANG" = "nodejs" ]; then
|
|
574
594
|
for f in migrations/*.js; do
|
|
575
595
|
[ -f "$f" ] || continue
|