@databricks/appkit 0.38.1 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/appkit/package.js +1 -1
- package/dist/cli/commands/generate-types.js +114 -5
- package/dist/cli/commands/generate-types.js.map +1 -1
- package/dist/cli/commands/spawn-lock.js +116 -0
- package/dist/cli/commands/spawn-lock.js.map +1 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/connectors/index.js +1 -1
- package/dist/connectors/sql-warehouse/client.js +168 -1
- package/dist/connectors/sql-warehouse/client.js.map +1 -1
- package/dist/connectors/sql-warehouse/index.js +1 -1
- package/dist/connectors/sql-warehouse/warehouse-poll-backoff.js +24 -0
- package/dist/connectors/sql-warehouse/warehouse-poll-backoff.js.map +1 -0
- package/dist/connectors/sql-warehouse/warehouse-status-emitter.js +40 -0
- package/dist/connectors/sql-warehouse/warehouse-status-emitter.js.map +1 -0
- package/dist/plugins/analytics/analytics.d.ts.map +1 -1
- package/dist/plugins/analytics/analytics.js +73 -8
- package/dist/plugins/analytics/analytics.js.map +1 -1
- package/dist/plugins/analytics/manifest.js +17 -5
- package/dist/plugins/analytics/types.d.ts +13 -0
- package/dist/plugins/analytics/types.d.ts.map +1 -1
- package/dist/plugins/analytics/types.js.map +1 -1
- package/dist/type-generator/index.js +115 -12
- package/dist/type-generator/index.js.map +1 -1
- package/dist/type-generator/preflight.js +24 -0
- package/dist/type-generator/preflight.js.map +1 -0
- package/dist/type-generator/query-registry.js +322 -90
- package/dist/type-generator/query-registry.js.map +1 -1
- package/dist/type-generator/types.js.map +1 -1
- package/dist/type-generator/vite-plugin.d.ts.map +1 -1
- package/dist/type-generator/vite-plugin.js +141 -7
- package/dist/type-generator/vite-plugin.js.map +1 -1
- package/dist/type-generator/warehouse-status.js +116 -0
- package/dist/type-generator/warehouse-status.js.map +1 -0
- package/docs/development/type-generation.md +13 -0
- package/docs/plugins/analytics.md +156 -0
- package/package.json +1 -1
- package/sbom.cdx.json +1 -1
package/dist/appkit/package.js
CHANGED
|
@@ -1,15 +1,33 @@
|
|
|
1
|
+
import { acquireSpawnLock, getSpawnLockPath, releaseSpawnLock } from "./spawn-lock.js";
|
|
1
2
|
import fs from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
|
-
import { Command } from "commander";
|
|
4
|
+
import { Command, Option } from "commander";
|
|
5
|
+
import { spawn } from "node:child_process";
|
|
4
6
|
|
|
5
7
|
//#region src/cli/commands/generate-types.ts
|
|
6
8
|
/**
|
|
7
|
-
*
|
|
9
|
+
* Resolve the typegen pre-flight mode for the CLI. Defaults to "non-blocking" —
|
|
10
|
+
* a one-shot CLI can't describe in the background, so by default it never
|
|
11
|
+
* describes at all: it skips the warehouse probe AND every DESCRIBE, emits
|
|
12
|
+
* best-available types (cache where the SQL hash matches, else `result: unknown`)
|
|
13
|
+
* and returns immediately, never blocking on — or failing because of — a
|
|
14
|
+
* warehouse, even a RUNNING one. Pass `--wait` (commander sets `wait: true`)
|
|
15
|
+
* for a deliberate/CI invocation that should wait for a starting warehouse and
|
|
16
|
+
* fail fast on a stopped one.
|
|
17
|
+
*/
|
|
18
|
+
function resolveTypegenMode(options) {
|
|
19
|
+
return options?.wait ? "blocking" : "non-blocking";
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Generate types command implementation. Runs the library generate (which, in
|
|
23
|
+
* non-blocking mode, writes degraded types and returns immediately). This is the
|
|
24
|
+
* SAME work the worker performs in blocking mode in the background.
|
|
8
25
|
*/
|
|
9
26
|
async function runGenerateTypes(rootDir, outFile, warehouseId, options) {
|
|
10
27
|
try {
|
|
11
28
|
const resolvedRootDir = rootDir || process.cwd();
|
|
12
29
|
const noCache = options?.noCache || false;
|
|
30
|
+
const mode = resolveTypegenMode(options);
|
|
13
31
|
const typeGen = await import("@databricks/appkit/type-generator");
|
|
14
32
|
const resolvedWarehouseId = warehouseId || process.env.DATABRICKS_WAREHOUSE_ID;
|
|
15
33
|
if (resolvedWarehouseId) {
|
|
@@ -20,7 +38,8 @@ async function runGenerateTypes(rootDir, outFile, warehouseId, options) {
|
|
|
20
38
|
queryFolder,
|
|
21
39
|
outFile: resolvedOutFile,
|
|
22
40
|
warehouseId: resolvedWarehouseId,
|
|
23
|
-
noCache
|
|
41
|
+
noCache,
|
|
42
|
+
mode
|
|
24
43
|
});
|
|
25
44
|
console.log(`Generated query types: ${resolvedOutFile}`);
|
|
26
45
|
}
|
|
@@ -37,15 +56,105 @@ async function runGenerateTypes(rootDir, outFile, warehouseId, options) {
|
|
|
37
56
|
console.error("Please install @databricks/appkit to use this command.");
|
|
38
57
|
process.exit(1);
|
|
39
58
|
}
|
|
59
|
+
if (error instanceof Error && (error.name === "TypegenSyntaxError" || error.name === "TypegenFatalError")) {
|
|
60
|
+
console.error(error.message);
|
|
61
|
+
process.exit(1);
|
|
62
|
+
}
|
|
40
63
|
throw error;
|
|
41
64
|
}
|
|
42
65
|
}
|
|
43
|
-
|
|
66
|
+
/**
|
|
67
|
+
* Spawn the detached blocking worker that refreshes real types in the background
|
|
68
|
+
* after the foreground non-blocking generate has already written degraded types.
|
|
69
|
+
*
|
|
70
|
+
* Re-invokes THIS CLI (`process.execPath` + `process.argv[1]` — the bin entry
|
|
71
|
+
* that launched us) with `generate-types --wait --worker-lock <lockPath>` plus
|
|
72
|
+
* the same positional target options the foreground used, so the worker writes
|
|
73
|
+
* to the same out file / reads the same query folder. The worker is:
|
|
74
|
+
* - `detached: true` + `.unref()` so it outlives this process (install/dev-setup
|
|
75
|
+
* can finish and exit while the worker keeps warming the warehouse).
|
|
76
|
+
* - `stdio: "ignore"` so it never holds the parent's pipes open or interleaves
|
|
77
|
+
* output into the install/dev log.
|
|
78
|
+
*
|
|
79
|
+
* Spawning is wrapped so any failure is non-fatal: the caller still has degraded
|
|
80
|
+
* types and exits 0.
|
|
81
|
+
*
|
|
82
|
+
* @param lockPath - the acquired single-flight lock; passed to the worker so it
|
|
83
|
+
* releases the SAME lock when it finishes.
|
|
84
|
+
* @param targets - the foreground's positional args, forwarded verbatim.
|
|
85
|
+
* @returns true if the worker was spawned, false if spawning threw.
|
|
86
|
+
*/
|
|
87
|
+
function spawnTypegenWorker(lockPath, targets) {
|
|
88
|
+
const cliEntry = process.argv[1];
|
|
89
|
+
const positionals = [];
|
|
90
|
+
for (const value of [
|
|
91
|
+
targets.rootDir,
|
|
92
|
+
targets.outFile,
|
|
93
|
+
targets.warehouseId
|
|
94
|
+
]) {
|
|
95
|
+
if (value === void 0) break;
|
|
96
|
+
positionals.push(value);
|
|
97
|
+
}
|
|
98
|
+
const args = [
|
|
99
|
+
...process.execArgv,
|
|
100
|
+
cliEntry,
|
|
101
|
+
"generate-types",
|
|
102
|
+
"--wait",
|
|
103
|
+
"--worker-lock",
|
|
104
|
+
lockPath,
|
|
105
|
+
...positionals
|
|
106
|
+
];
|
|
107
|
+
try {
|
|
108
|
+
spawn(process.execPath, args, {
|
|
109
|
+
detached: true,
|
|
110
|
+
stdio: "ignore"
|
|
111
|
+
}).unref();
|
|
112
|
+
return true;
|
|
113
|
+
} catch (error) {
|
|
114
|
+
console.error(`Could not start background type refresh: ${error instanceof Error ? error.message : String(error)}`);
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The command action. Orchestrates the non-blocking foreground contract:
|
|
120
|
+
* 1. Run the library generate (writes degraded types immediately in non-blocking
|
|
121
|
+
* mode; does the full blocking lifecycle when this is the worker).
|
|
122
|
+
* 2. If this is a non-blocking, non-worker invocation, try to spawn the detached
|
|
123
|
+
* blocking worker behind the single-flight lock. If the lock is already held
|
|
124
|
+
* by a live worker, skip (single-flight) with a one-line note. Either way the
|
|
125
|
+
* foreground returns normally (exit 0).
|
|
126
|
+
* 3. If this IS the worker (`--worker-lock` present), it ran blocking above and
|
|
127
|
+
* releases the lock here (and via a process-exit guard, so a hard failure /
|
|
128
|
+
* process.exit still frees it).
|
|
129
|
+
*/
|
|
130
|
+
async function generateTypesAction(rootDir, outFile, warehouseId, options) {
|
|
131
|
+
const isWorker = typeof options.workerLock === "string";
|
|
132
|
+
if (isWorker && options.workerLock) {
|
|
133
|
+
const lockPath = options.workerLock;
|
|
134
|
+
process.once("exit", () => releaseSpawnLock(lockPath));
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
await runGenerateTypes(rootDir, outFile, warehouseId, options);
|
|
138
|
+
} finally {
|
|
139
|
+
if (isWorker && options.workerLock) releaseSpawnLock(options.workerLock);
|
|
140
|
+
}
|
|
141
|
+
if (!isWorker && resolveTypegenMode(options) === "non-blocking") {
|
|
142
|
+
const lockPath = getSpawnLockPath(rootDir || process.cwd());
|
|
143
|
+
if (acquireSpawnLock(lockPath)) spawnTypegenWorker(lockPath, {
|
|
144
|
+
rootDir,
|
|
145
|
+
outFile,
|
|
146
|
+
warehouseId
|
|
147
|
+
});
|
|
148
|
+
else console.log("Type refresh already in progress, skipping.");
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const generateTypesCommand = new Command("generate-types").description("Generate TypeScript types from SQL queries").argument("[rootDir]", "Root directory of the project", process.cwd()).argument("[outFile]", "Output file path", path.join(process.cwd(), "shared/appkit-types/analytics.d.ts")).argument("[warehouseId]", "Databricks warehouse ID").option("--no-cache", "Disable caching for type generation").option("--wait", "Wait for warehouse readiness instead of degrading (use for CI)").addOption(new Option("--worker-lock <path>", "Internal: detached worker lock path").hideHelp()).addHelpText("after", `
|
|
44
152
|
Examples:
|
|
45
153
|
$ appkit generate-types
|
|
46
154
|
$ appkit generate-types . shared/appkit-types/analytics.d.ts
|
|
47
155
|
$ appkit generate-types . shared/appkit-types/analytics.d.ts my-warehouse-id
|
|
48
|
-
$ appkit generate-types --no-cache
|
|
156
|
+
$ appkit generate-types --no-cache
|
|
157
|
+
$ appkit generate-types --wait # CI: wait for the warehouse and fail on a cold one`).action(generateTypesAction);
|
|
49
158
|
|
|
50
159
|
//#endregion
|
|
51
160
|
export { generateTypesCommand };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generate-types.js","names":[],"sources":["../../../src/cli/commands/generate-types.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport path from \"node:path\";\nimport { Command } from \"commander\";\n\n/**\n * Generate types command implementation\n */\nasync function runGenerateTypes(\n rootDir?: string,\n outFile?: string,\n warehouseId?: string,\n options?: { noCache?: boolean },\n) {\n try {\n const resolvedRootDir = rootDir || process.cwd();\n const noCache = options?.noCache || false;\n\n const typeGen = await import(\"@databricks/appkit/type-generator\");\n\n // Generate analytics query types (requires warehouse ID)\n const resolvedWarehouseId =\n warehouseId || process.env.DATABRICKS_WAREHOUSE_ID;\n\n if (resolvedWarehouseId) {\n const resolvedOutFile =\n outFile ||\n path.join(process.cwd(), \"shared/appkit-types/analytics.d.ts\");\n\n const queryFolder = path.join(resolvedRootDir, \"config/queries\");\n if (fs.existsSync(queryFolder)) {\n await typeGen.generateFromEntryPoint({\n queryFolder,\n outFile: resolvedOutFile,\n warehouseId: resolvedWarehouseId,\n noCache,\n });\n console.log(`Generated query types: ${resolvedOutFile}`);\n }\n } else {\n console.error(\n \"Skipping query type generation: no warehouse ID. Set DATABRICKS_WAREHOUSE_ID or pass as argument.\",\n );\n }\n\n // Generate serving endpoint types (no warehouse required)\n const servingOutFile = path.join(\n process.cwd(),\n \"shared/appkit-types/serving.d.ts\",\n );\n await typeGen.generateServingTypes({\n outFile: servingOutFile,\n noCache,\n });\n console.log(`Generated serving types: ${servingOutFile}`);\n } catch (error) {\n if (\n error instanceof Error &&\n error.message.includes(\"Cannot find module\")\n ) {\n console.error(\n \"Error: The 'generate-types' command is only available in @databricks/appkit.\",\n );\n console.error(\"Please install @databricks/appkit to use this command.\");\n process.exit(1);\n }\n throw error;\n }\n}\n\nexport const generateTypesCommand = new Command(\"generate-types\")\n .description(\"Generate TypeScript types from SQL queries\")\n .argument(\"[rootDir]\", \"Root directory of the project\", process.cwd())\n .argument(\n \"[outFile]\",\n \"Output file path\",\n path.join(process.cwd(), \"shared/appkit-types/analytics.d.ts\"),\n )\n .argument(\"[warehouseId]\", \"Databricks warehouse ID\")\n .option(\"--no-cache\", \"Disable caching for type generation\")\n .addHelpText(\n \"after\",\n `\nExamples:\n $ appkit generate-types\n $ appkit generate-types . shared/appkit-types/analytics.d.ts\n $ appkit generate-types . shared/appkit-types/analytics.d.ts my-warehouse-id\n $ appkit generate-types --no-cache`,\n )\n .action(runGenerateTypes);\n"],"mappings":";;;;;;;;AAOA,eAAe,iBACb,SACA,SACA,aACA,SACA;AACA,KAAI;EACF,MAAM,kBAAkB,WAAW,QAAQ,KAAK;EAChD,MAAM,UAAU,SAAS,WAAW;EAEpC,MAAM,UAAU,MAAM,OAAO;EAG7B,MAAM,sBACJ,eAAe,QAAQ,IAAI;AAE7B,MAAI,qBAAqB;GACvB,MAAM,kBACJ,WACA,KAAK,KAAK,QAAQ,KAAK,EAAE,qCAAqC;GAEhE,MAAM,cAAc,KAAK,KAAK,iBAAiB,iBAAiB;AAChE,OAAI,GAAG,WAAW,YAAY,EAAE;AAC9B,UAAM,QAAQ,uBAAuB;KACnC;KACA,SAAS;KACT,aAAa;KACb;KACD,CAAC;AACF,YAAQ,IAAI,0BAA0B,kBAAkB;;QAG1D,SAAQ,MACN,oGACD;EAIH,MAAM,iBAAiB,KAAK,KAC1B,QAAQ,KAAK,EACb,mCACD;AACD,QAAM,QAAQ,qBAAqB;GACjC,SAAS;GACT;GACD,CAAC;AACF,UAAQ,IAAI,4BAA4B,iBAAiB;UAClD,OAAO;AACd,MACE,iBAAiB,SACjB,MAAM,QAAQ,SAAS,qBAAqB,EAC5C;AACA,WAAQ,MACN,+EACD;AACD,WAAQ,MAAM,yDAAyD;AACvE,WAAQ,KAAK,EAAE;;AAEjB,QAAM;;;AAIV,MAAa,uBAAuB,IAAI,QAAQ,iBAAiB,CAC9D,YAAY,6CAA6C,CACzD,SAAS,aAAa,iCAAiC,QAAQ,KAAK,CAAC,CACrE,SACC,aACA,oBACA,KAAK,KAAK,QAAQ,KAAK,EAAE,qCAAqC,CAC/D,CACA,SAAS,iBAAiB,0BAA0B,CACpD,OAAO,cAAc,sCAAsC,CAC3D,YACC,SACA;;;;;sCAMD,CACA,OAAO,iBAAiB"}
|
|
1
|
+
{"version":3,"file":"generate-types.js","names":[],"sources":["../../../src/cli/commands/generate-types.ts"],"sourcesContent":["import { spawn } from \"node:child_process\";\nimport fs from \"node:fs\";\nimport path from \"node:path\";\nimport { Command, Option } from \"commander\";\nimport {\n acquireSpawnLock,\n getSpawnLockPath,\n releaseSpawnLock,\n} from \"./spawn-lock.js\";\n\n/**\n * Resolve the typegen pre-flight mode for the CLI. Defaults to \"non-blocking\" —\n * a one-shot CLI can't describe in the background, so by default it never\n * describes at all: it skips the warehouse probe AND every DESCRIBE, emits\n * best-available types (cache where the SQL hash matches, else `result: unknown`)\n * and returns immediately, never blocking on — or failing because of — a\n * warehouse, even a RUNNING one. Pass `--wait` (commander sets `wait: true`)\n * for a deliberate/CI invocation that should wait for a starting warehouse and\n * fail fast on a stopped one.\n */\nexport function resolveTypegenMode(options?: {\n wait?: boolean;\n}): \"non-blocking\" | \"blocking\" {\n return options?.wait ? \"blocking\" : \"non-blocking\";\n}\n\n/** Options parsed by commander for the generate-types command. */\ninterface GenerateTypesOptions {\n noCache?: boolean;\n wait?: boolean;\n /**\n * Internal: present only on the detached worker invocation. Carries the path\n * of the single-flight lock this worker must release when it finishes. Its\n * presence is what marks an invocation as \"the worker\" — workers always run\n * with `--wait`, so they never spawn another worker (only non-blocking runs\n * spawn), which terminates the recursion.\n */\n workerLock?: string;\n}\n\n/**\n * Generate types command implementation. Runs the library generate (which, in\n * non-blocking mode, writes degraded types and returns immediately). This is the\n * SAME work the worker performs in blocking mode in the background.\n */\nasync function runGenerateTypes(\n rootDir?: string,\n outFile?: string,\n warehouseId?: string,\n options?: GenerateTypesOptions,\n) {\n try {\n const resolvedRootDir = rootDir || process.cwd();\n const noCache = options?.noCache || false;\n const mode = resolveTypegenMode(options);\n\n const typeGen = await import(\"@databricks/appkit/type-generator\");\n\n // Generate analytics query types (requires warehouse ID)\n const resolvedWarehouseId =\n warehouseId || process.env.DATABRICKS_WAREHOUSE_ID;\n\n if (resolvedWarehouseId) {\n const resolvedOutFile =\n outFile ||\n path.join(process.cwd(), \"shared/appkit-types/analytics.d.ts\");\n\n const queryFolder = path.join(resolvedRootDir, \"config/queries\");\n if (fs.existsSync(queryFolder)) {\n await typeGen.generateFromEntryPoint({\n queryFolder,\n outFile: resolvedOutFile,\n warehouseId: resolvedWarehouseId,\n noCache,\n mode,\n });\n console.log(`Generated query types: ${resolvedOutFile}`);\n }\n } else {\n console.error(\n \"Skipping query type generation: no warehouse ID. Set DATABRICKS_WAREHOUSE_ID or pass as argument.\",\n );\n }\n\n // Generate serving endpoint types (no warehouse required)\n const servingOutFile = path.join(\n process.cwd(),\n \"shared/appkit-types/serving.d.ts\",\n );\n await typeGen.generateServingTypes({\n outFile: servingOutFile,\n noCache,\n });\n console.log(`Generated serving types: ${servingOutFile}`);\n } catch (error) {\n if (\n error instanceof Error &&\n error.message.includes(\"Cannot find module\")\n ) {\n console.error(\n \"Error: The 'generate-types' command is only available in @databricks/appkit.\",\n );\n console.error(\"Please install @databricks/appkit to use this command.\");\n process.exit(1);\n }\n // TypegenSyntaxError / TypegenFatalError carry a complete, actionable\n // message (which queries failed and how to debug them). The stack trace\n // points into appkit internals and is noise for app developers, so print\n // only the message and exit non-zero instead of letting it bubble up.\n if (\n error instanceof Error &&\n (error.name === \"TypegenSyntaxError\" ||\n error.name === \"TypegenFatalError\")\n ) {\n console.error(error.message);\n process.exit(1);\n }\n throw error;\n }\n}\n\n/**\n * Spawn the detached blocking worker that refreshes real types in the background\n * after the foreground non-blocking generate has already written degraded types.\n *\n * Re-invokes THIS CLI (`process.execPath` + `process.argv[1]` — the bin entry\n * that launched us) with `generate-types --wait --worker-lock <lockPath>` plus\n * the same positional target options the foreground used, so the worker writes\n * to the same out file / reads the same query folder. The worker is:\n * - `detached: true` + `.unref()` so it outlives this process (install/dev-setup\n * can finish and exit while the worker keeps warming the warehouse).\n * - `stdio: \"ignore\"` so it never holds the parent's pipes open or interleaves\n * output into the install/dev log.\n *\n * Spawning is wrapped so any failure is non-fatal: the caller still has degraded\n * types and exits 0.\n *\n * @param lockPath - the acquired single-flight lock; passed to the worker so it\n * releases the SAME lock when it finishes.\n * @param targets - the foreground's positional args, forwarded verbatim.\n * @returns true if the worker was spawned, false if spawning threw.\n */\nexport function spawnTypegenWorker(\n lockPath: string,\n targets: { rootDir?: string; outFile?: string; warehouseId?: string },\n): boolean {\n // The script the runtime launched us with (the `appkit` bin shim). Re-running\n // it under the same node binary reproduces this exact CLI in the worker.\n const cliEntry = process.argv[1];\n\n // Forward the positionals in declaration order (rootDir, outFile,\n // warehouseId). Stop at the first undefined so we never pass a literal\n // \"undefined\" — commander would treat it as a positional value. (rootDir is\n // effectively always set by commander's default, but guard anyway.)\n const positionals: string[] = [];\n for (const value of [targets.rootDir, targets.outFile, targets.warehouseId]) {\n if (value === undefined) break;\n positionals.push(value);\n }\n\n const args = [\n // Forward the parent's node/loader flags so the worker runs under the same\n // runtime. Critically this carries tsx's `--require`/`--import` when the CLI\n // is run from source (`tsx index.ts …`); without them the worker would be\n // `node index.ts …`, which can't parse TypeScript and dies silently — the\n // degraded types would then never refresh. Empty for the built bin (plain\n // `node bin/appkit.js`), so production behaviour is unchanged.\n ...process.execArgv,\n cliEntry,\n \"generate-types\",\n \"--wait\",\n \"--worker-lock\",\n lockPath,\n ...positionals,\n ];\n\n try {\n const child = spawn(process.execPath, args, {\n detached: true,\n stdio: \"ignore\",\n });\n child.unref();\n return true;\n } catch (error) {\n // Non-fatal: the foreground already wrote degraded types. Log and move on.\n console.error(\n `Could not start background type refresh: ${\n error instanceof Error ? error.message : String(error)\n }`,\n );\n return false;\n }\n}\n\n/**\n * The command action. Orchestrates the non-blocking foreground contract:\n * 1. Run the library generate (writes degraded types immediately in non-blocking\n * mode; does the full blocking lifecycle when this is the worker).\n * 2. If this is a non-blocking, non-worker invocation, try to spawn the detached\n * blocking worker behind the single-flight lock. If the lock is already held\n * by a live worker, skip (single-flight) with a one-line note. Either way the\n * foreground returns normally (exit 0).\n * 3. If this IS the worker (`--worker-lock` present), it ran blocking above and\n * releases the lock here (and via a process-exit guard, so a hard failure /\n * process.exit still frees it).\n */\nasync function generateTypesAction(\n rootDir: string | undefined,\n outFile: string | undefined,\n warehouseId: string | undefined,\n options: GenerateTypesOptions,\n) {\n const isWorker = typeof options.workerLock === \"string\";\n\n // A worker must always free its lock, even if the blocking generate throws or\n // calls process.exit (TypegenFatalError → exit 1). The exit handler covers the\n // process.exit / uncaught paths; the finally covers the normal return.\n if (isWorker && options.workerLock) {\n const lockPath = options.workerLock;\n process.once(\"exit\", () => releaseSpawnLock(lockPath));\n }\n\n try {\n await runGenerateTypes(rootDir, outFile, warehouseId, options);\n } finally {\n if (isWorker && options.workerLock) {\n releaseSpawnLock(options.workerLock);\n }\n }\n\n // Only a non-blocking, non-worker invocation spawns. A worker is always\n // --wait (so resolveTypegenMode → \"blocking\"), which both prevents recursion\n // and means we never get here for a worker.\n if (!isWorker && resolveTypegenMode(options) === \"non-blocking\") {\n const resolvedRootDir = rootDir || process.cwd();\n const lockPath = getSpawnLockPath(resolvedRootDir);\n\n if (acquireSpawnLock(lockPath)) {\n spawnTypegenWorker(lockPath, { rootDir, outFile, warehouseId });\n } else {\n console.log(\"Type refresh already in progress, skipping.\");\n }\n }\n}\n\nexport const generateTypesCommand = new Command(\"generate-types\")\n .description(\"Generate TypeScript types from SQL queries\")\n .argument(\"[rootDir]\", \"Root directory of the project\", process.cwd())\n .argument(\n \"[outFile]\",\n \"Output file path\",\n path.join(process.cwd(), \"shared/appkit-types/analytics.d.ts\"),\n )\n .argument(\"[warehouseId]\", \"Databricks warehouse ID\")\n .option(\"--no-cache\", \"Disable caching for type generation\")\n .option(\n \"--wait\",\n \"Wait for warehouse readiness instead of degrading (use for CI)\",\n )\n // Internal: marks the detached background worker and carries the lock it must\n // release. Hidden from --help; users should never pass it.\n .addOption(\n new Option(\n \"--worker-lock <path>\",\n \"Internal: detached worker lock path\",\n ).hideHelp(),\n )\n .addHelpText(\n \"after\",\n `\nExamples:\n $ appkit generate-types\n $ appkit generate-types . shared/appkit-types/analytics.d.ts\n $ appkit generate-types . shared/appkit-types/analytics.d.ts my-warehouse-id\n $ appkit generate-types --no-cache\n $ appkit generate-types --wait # CI: wait for the warehouse and fail on a cold one`,\n )\n .action(generateTypesAction);\n"],"mappings":";;;;;;;;;;;;;;;;;AAoBA,SAAgB,mBAAmB,SAEH;AAC9B,QAAO,SAAS,OAAO,aAAa;;;;;;;AAsBtC,eAAe,iBACb,SACA,SACA,aACA,SACA;AACA,KAAI;EACF,MAAM,kBAAkB,WAAW,QAAQ,KAAK;EAChD,MAAM,UAAU,SAAS,WAAW;EACpC,MAAM,OAAO,mBAAmB,QAAQ;EAExC,MAAM,UAAU,MAAM,OAAO;EAG7B,MAAM,sBACJ,eAAe,QAAQ,IAAI;AAE7B,MAAI,qBAAqB;GACvB,MAAM,kBACJ,WACA,KAAK,KAAK,QAAQ,KAAK,EAAE,qCAAqC;GAEhE,MAAM,cAAc,KAAK,KAAK,iBAAiB,iBAAiB;AAChE,OAAI,GAAG,WAAW,YAAY,EAAE;AAC9B,UAAM,QAAQ,uBAAuB;KACnC;KACA,SAAS;KACT,aAAa;KACb;KACA;KACD,CAAC;AACF,YAAQ,IAAI,0BAA0B,kBAAkB;;QAG1D,SAAQ,MACN,oGACD;EAIH,MAAM,iBAAiB,KAAK,KAC1B,QAAQ,KAAK,EACb,mCACD;AACD,QAAM,QAAQ,qBAAqB;GACjC,SAAS;GACT;GACD,CAAC;AACF,UAAQ,IAAI,4BAA4B,iBAAiB;UAClD,OAAO;AACd,MACE,iBAAiB,SACjB,MAAM,QAAQ,SAAS,qBAAqB,EAC5C;AACA,WAAQ,MACN,+EACD;AACD,WAAQ,MAAM,yDAAyD;AACvE,WAAQ,KAAK,EAAE;;AAMjB,MACE,iBAAiB,UAChB,MAAM,SAAS,wBACd,MAAM,SAAS,sBACjB;AACA,WAAQ,MAAM,MAAM,QAAQ;AAC5B,WAAQ,KAAK,EAAE;;AAEjB,QAAM;;;;;;;;;;;;;;;;;;;;;;;;AAyBV,SAAgB,mBACd,UACA,SACS;CAGT,MAAM,WAAW,QAAQ,KAAK;CAM9B,MAAM,cAAwB,EAAE;AAChC,MAAK,MAAM,SAAS;EAAC,QAAQ;EAAS,QAAQ;EAAS,QAAQ;EAAY,EAAE;AAC3E,MAAI,UAAU,OAAW;AACzB,cAAY,KAAK,MAAM;;CAGzB,MAAM,OAAO;EAOX,GAAG,QAAQ;EACX;EACA;EACA;EACA;EACA;EACA,GAAG;EACJ;AAED,KAAI;AAKF,EAJc,MAAM,QAAQ,UAAU,MAAM;GAC1C,UAAU;GACV,OAAO;GACR,CAAC,CACI,OAAO;AACb,SAAO;UACA,OAAO;AAEd,UAAQ,MACN,4CACE,iBAAiB,QAAQ,MAAM,UAAU,OAAO,MAAM,GAEzD;AACD,SAAO;;;;;;;;;;;;;;;AAgBX,eAAe,oBACb,SACA,SACA,aACA,SACA;CACA,MAAM,WAAW,OAAO,QAAQ,eAAe;AAK/C,KAAI,YAAY,QAAQ,YAAY;EAClC,MAAM,WAAW,QAAQ;AACzB,UAAQ,KAAK,cAAc,iBAAiB,SAAS,CAAC;;AAGxD,KAAI;AACF,QAAM,iBAAiB,SAAS,SAAS,aAAa,QAAQ;WACtD;AACR,MAAI,YAAY,QAAQ,WACtB,kBAAiB,QAAQ,WAAW;;AAOxC,KAAI,CAAC,YAAY,mBAAmB,QAAQ,KAAK,gBAAgB;EAE/D,MAAM,WAAW,iBADO,WAAW,QAAQ,KAAK,CACE;AAElD,MAAI,iBAAiB,SAAS,CAC5B,oBAAmB,UAAU;GAAE;GAAS;GAAS;GAAa,CAAC;MAE/D,SAAQ,IAAI,8CAA8C;;;AAKhE,MAAa,uBAAuB,IAAI,QAAQ,iBAAiB,CAC9D,YAAY,6CAA6C,CACzD,SAAS,aAAa,iCAAiC,QAAQ,KAAK,CAAC,CACrE,SACC,aACA,oBACA,KAAK,KAAK,QAAQ,KAAK,EAAE,qCAAqC,CAC/D,CACA,SAAS,iBAAiB,0BAA0B,CACpD,OAAO,cAAc,sCAAsC,CAC3D,OACC,UACA,iEACD,CAGA,UACC,IAAI,OACF,wBACA,sCACD,CAAC,UAAU,CACb,CACA,YACC,SACA;;;;;;wFAOD,CACA,OAAO,oBAAoB"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
//#region src/cli/commands/spawn-lock.ts
|
|
5
|
+
/**
|
|
6
|
+
* How long a spawn lock is considered fresh. A held lock newer than this means a
|
|
7
|
+
* background worker is genuinely in flight, so the foreground skips spawning;
|
|
8
|
+
* older than this the lock is presumed orphaned (the worker crashed/was killed
|
|
9
|
+
* before it could release) and is stolen.
|
|
10
|
+
*
|
|
11
|
+
* Must comfortably exceed the worker's worst-case runtime: the blocking preflight
|
|
12
|
+
* wait cap (PREFLIGHT_WAIT_MAX_MS = 5 min in the type-generator) plus a DESCRIBE
|
|
13
|
+
* budget. Six minutes leaves ~1 min of headroom over a worker that waits the full
|
|
14
|
+
* preflight window and then describes.
|
|
15
|
+
*/
|
|
16
|
+
const SPAWN_LOCK_STALE_MS = 360 * 1e3;
|
|
17
|
+
/**
|
|
18
|
+
* Resolve the on-disk path of the single-flight spawn lock for a project.
|
|
19
|
+
*
|
|
20
|
+
* Lives alongside the type-generator cache (`node_modules/.databricks/appkit/`)
|
|
21
|
+
* so it shares the same already-creatable, gitignored, per-project location and
|
|
22
|
+
* doesn't introduce a new directory. The lock is keyed only by project root, so
|
|
23
|
+
* concurrent `generate-types` invocations for the same project (postinstall +
|
|
24
|
+
* predev, say) contend for one lock and only one wins the spawn.
|
|
25
|
+
*
|
|
26
|
+
* @param rootDir - project root (the resolved first CLI argument / cwd).
|
|
27
|
+
* @returns absolute path to the lock file.
|
|
28
|
+
*/
|
|
29
|
+
function getSpawnLockPath(rootDir) {
|
|
30
|
+
return path.join(rootDir, "node_modules", ".databricks", "appkit", ".appkit-typegen-worker.lock");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Try to acquire the single-flight spawn lock.
|
|
34
|
+
*
|
|
35
|
+
* Atomic create via `fs.writeFileSync(lockPath, ..., { flag: "wx" })` — `wx`
|
|
36
|
+
* fails (EEXIST) if the file already exists, so the create itself is the
|
|
37
|
+
* mutual-exclusion primitive (no check-then-create race between two foreground
|
|
38
|
+
* processes). The lock body records pid + timestamp purely for debugging.
|
|
39
|
+
*
|
|
40
|
+
* On EEXIST we stat the existing lock:
|
|
41
|
+
* - fresh (mtime within {@link staleMs}) → a worker is in flight, return false.
|
|
42
|
+
* - stale (mtime older than staleMs) → presumed orphaned; unlink and recreate.
|
|
43
|
+
* The recreate also uses `wx`, so if a competing process steals it first we
|
|
44
|
+
* lose the race cleanly and return false.
|
|
45
|
+
*
|
|
46
|
+
* Any unexpected error (permission, ENOENT on a missing parent dir we couldn't
|
|
47
|
+
* create, …) is swallowed and reported as "not acquired": failing to take the
|
|
48
|
+
* lock must never break the foreground — at worst we skip the background refresh.
|
|
49
|
+
*
|
|
50
|
+
* @param lockPath - path returned by {@link getSpawnLockPath}.
|
|
51
|
+
* @param staleMs - age beyond which a held lock is stolen. Defaults to
|
|
52
|
+
* {@link SPAWN_LOCK_STALE_MS}.
|
|
53
|
+
* @returns true if this caller now owns the lock (and must release it), false if
|
|
54
|
+
* another live worker holds it or the lock couldn't be taken.
|
|
55
|
+
*/
|
|
56
|
+
function acquireSpawnLock(lockPath, staleMs = SPAWN_LOCK_STALE_MS) {
|
|
57
|
+
const body = `${process.pid} ${Date.now()}\n`;
|
|
58
|
+
try {
|
|
59
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
60
|
+
} catch {}
|
|
61
|
+
try {
|
|
62
|
+
fs.writeFileSync(lockPath, body, { flag: "wx" });
|
|
63
|
+
return true;
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (!isErrnoException(error) || error.code !== "EEXIST") return false;
|
|
66
|
+
}
|
|
67
|
+
let mtimeMs;
|
|
68
|
+
try {
|
|
69
|
+
mtimeMs = fs.statSync(lockPath).mtimeMs;
|
|
70
|
+
} catch {
|
|
71
|
+
return tryCreate(lockPath, body);
|
|
72
|
+
}
|
|
73
|
+
if (Date.now() - mtimeMs < staleMs) return false;
|
|
74
|
+
try {
|
|
75
|
+
fs.unlinkSync(lockPath);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (isErrnoException(error) && error.code !== "ENOENT") return false;
|
|
78
|
+
}
|
|
79
|
+
return tryCreate(lockPath, body);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Release the spawn lock. Unlink, ignoring ENOENT (already gone — e.g. it was
|
|
83
|
+
* stolen as stale by another process, or never existed). Any other error is
|
|
84
|
+
* swallowed: a failed release at worst leaves a stale lock that the next caller
|
|
85
|
+
* will steal after {@link SPAWN_LOCK_STALE_MS}.
|
|
86
|
+
*
|
|
87
|
+
* @param lockPath - path returned by {@link getSpawnLockPath}.
|
|
88
|
+
*/
|
|
89
|
+
function releaseSpawnLock(lockPath) {
|
|
90
|
+
try {
|
|
91
|
+
fs.unlinkSync(lockPath);
|
|
92
|
+
} catch {}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Attempt an atomic `wx` create, returning whether it succeeded. EEXIST (a
|
|
96
|
+
* racing creator beat us) and any other error map to false.
|
|
97
|
+
*/
|
|
98
|
+
function tryCreate(lockPath, body) {
|
|
99
|
+
try {
|
|
100
|
+
fs.writeFileSync(lockPath, body, { flag: "wx" });
|
|
101
|
+
return true;
|
|
102
|
+
} catch {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Narrow an unknown caught value to a Node errno exception so `.code` is safe to
|
|
108
|
+
* read.
|
|
109
|
+
*/
|
|
110
|
+
function isErrnoException(error) {
|
|
111
|
+
return error instanceof Error && "code" in error;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
//#endregion
|
|
115
|
+
export { acquireSpawnLock, getSpawnLockPath, releaseSpawnLock };
|
|
116
|
+
//# sourceMappingURL=spawn-lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"spawn-lock.js","names":[],"sources":["../../../src/cli/commands/spawn-lock.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport path from \"node:path\";\n\n/**\n * How long a spawn lock is considered fresh. A held lock newer than this means a\n * background worker is genuinely in flight, so the foreground skips spawning;\n * older than this the lock is presumed orphaned (the worker crashed/was killed\n * before it could release) and is stolen.\n *\n * Must comfortably exceed the worker's worst-case runtime: the blocking preflight\n * wait cap (PREFLIGHT_WAIT_MAX_MS = 5 min in the type-generator) plus a DESCRIBE\n * budget. Six minutes leaves ~1 min of headroom over a worker that waits the full\n * preflight window and then describes.\n */\nexport const SPAWN_LOCK_STALE_MS = 6 * 60 * 1000;\n\n/**\n * Resolve the on-disk path of the single-flight spawn lock for a project.\n *\n * Lives alongside the type-generator cache (`node_modules/.databricks/appkit/`)\n * so it shares the same already-creatable, gitignored, per-project location and\n * doesn't introduce a new directory. The lock is keyed only by project root, so\n * concurrent `generate-types` invocations for the same project (postinstall +\n * predev, say) contend for one lock and only one wins the spawn.\n *\n * @param rootDir - project root (the resolved first CLI argument / cwd).\n * @returns absolute path to the lock file.\n */\nexport function getSpawnLockPath(rootDir: string): string {\n return path.join(\n rootDir,\n \"node_modules\",\n \".databricks\",\n \"appkit\",\n \".appkit-typegen-worker.lock\",\n );\n}\n\n/**\n * Try to acquire the single-flight spawn lock.\n *\n * Atomic create via `fs.writeFileSync(lockPath, ..., { flag: \"wx\" })` — `wx`\n * fails (EEXIST) if the file already exists, so the create itself is the\n * mutual-exclusion primitive (no check-then-create race between two foreground\n * processes). The lock body records pid + timestamp purely for debugging.\n *\n * On EEXIST we stat the existing lock:\n * - fresh (mtime within {@link staleMs}) → a worker is in flight, return false.\n * - stale (mtime older than staleMs) → presumed orphaned; unlink and recreate.\n * The recreate also uses `wx`, so if a competing process steals it first we\n * lose the race cleanly and return false.\n *\n * Any unexpected error (permission, ENOENT on a missing parent dir we couldn't\n * create, …) is swallowed and reported as \"not acquired\": failing to take the\n * lock must never break the foreground — at worst we skip the background refresh.\n *\n * @param lockPath - path returned by {@link getSpawnLockPath}.\n * @param staleMs - age beyond which a held lock is stolen. Defaults to\n * {@link SPAWN_LOCK_STALE_MS}.\n * @returns true if this caller now owns the lock (and must release it), false if\n * another live worker holds it or the lock couldn't be taken.\n */\nexport function acquireSpawnLock(\n lockPath: string,\n staleMs: number = SPAWN_LOCK_STALE_MS,\n): boolean {\n const body = `${process.pid} ${Date.now()}\\n`;\n\n try {\n fs.mkdirSync(path.dirname(lockPath), { recursive: true });\n } catch {\n // Parent dir creation is best-effort; the create below will surface any real\n // problem and we'll treat it as \"not acquired\".\n }\n\n try {\n fs.writeFileSync(lockPath, body, { flag: \"wx\" });\n return true;\n } catch (error) {\n if (!isErrnoException(error) || error.code !== \"EEXIST\") {\n // Unexpected failure — don't let lock IO break the foreground.\n return false;\n }\n }\n\n // Lock exists — decide fresh vs stale.\n let mtimeMs: number;\n try {\n mtimeMs = fs.statSync(lockPath).mtimeMs;\n } catch {\n // It vanished between the failed create and the stat (released by the\n // worker). Try once more to take it.\n return tryCreate(lockPath, body);\n }\n\n if (Date.now() - mtimeMs < staleMs) {\n // A worker is genuinely in flight.\n return false;\n }\n\n // Stale: steal it. Unlink (ignore ENOENT — someone else may have cleaned up)\n // then re-create with `wx` so we still lose cleanly to a racing stealer.\n try {\n fs.unlinkSync(lockPath);\n } catch (error) {\n if (isErrnoException(error) && error.code !== \"ENOENT\") {\n return false;\n }\n }\n return tryCreate(lockPath, body);\n}\n\n/**\n * Release the spawn lock. Unlink, ignoring ENOENT (already gone — e.g. it was\n * stolen as stale by another process, or never existed). Any other error is\n * swallowed: a failed release at worst leaves a stale lock that the next caller\n * will steal after {@link SPAWN_LOCK_STALE_MS}.\n *\n * @param lockPath - path returned by {@link getSpawnLockPath}.\n */\nexport function releaseSpawnLock(lockPath: string): void {\n try {\n fs.unlinkSync(lockPath);\n } catch {\n // ENOENT or any other error — releasing is best-effort.\n }\n}\n\n/**\n * Attempt an atomic `wx` create, returning whether it succeeded. EEXIST (a\n * racing creator beat us) and any other error map to false.\n */\nfunction tryCreate(lockPath: string, body: string): boolean {\n try {\n fs.writeFileSync(lockPath, body, { flag: \"wx\" });\n return true;\n } catch {\n return false;\n }\n}\n\n/**\n * Narrow an unknown caught value to a Node errno exception so `.code` is safe to\n * read.\n */\nfunction isErrnoException(error: unknown): error is NodeJS.ErrnoException {\n return error instanceof Error && \"code\" in error;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAcA,MAAa,sBAAsB,MAAS;;;;;;;;;;;;;AAc5C,SAAgB,iBAAiB,SAAyB;AACxD,QAAO,KAAK,KACV,SACA,gBACA,eACA,UACA,8BACD;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BH,SAAgB,iBACd,UACA,UAAkB,qBACT;CACT,MAAM,OAAO,GAAG,QAAQ,IAAI,GAAG,KAAK,KAAK,CAAC;AAE1C,KAAI;AACF,KAAG,UAAU,KAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,MAAM,CAAC;SACnD;AAKR,KAAI;AACF,KAAG,cAAc,UAAU,MAAM,EAAE,MAAM,MAAM,CAAC;AAChD,SAAO;UACA,OAAO;AACd,MAAI,CAAC,iBAAiB,MAAM,IAAI,MAAM,SAAS,SAE7C,QAAO;;CAKX,IAAI;AACJ,KAAI;AACF,YAAU,GAAG,SAAS,SAAS,CAAC;SAC1B;AAGN,SAAO,UAAU,UAAU,KAAK;;AAGlC,KAAI,KAAK,KAAK,GAAG,UAAU,QAEzB,QAAO;AAKT,KAAI;AACF,KAAG,WAAW,SAAS;UAChB,OAAO;AACd,MAAI,iBAAiB,MAAM,IAAI,MAAM,SAAS,SAC5C,QAAO;;AAGX,QAAO,UAAU,UAAU,KAAK;;;;;;;;;;AAWlC,SAAgB,iBAAiB,UAAwB;AACvD,KAAI;AACF,KAAG,WAAW,SAAS;SACjB;;;;;;AASV,SAAS,UAAU,UAAkB,MAAuB;AAC1D,KAAI;AACF,KAAG,cAAc,UAAU,MAAM,EAAE,MAAM,MAAM,CAAC;AAChD,SAAO;SACD;AACN,SAAO;;;;;;;AAQX,SAAS,iBAAiB,OAAgD;AACxE,QAAO,iBAAiB,SAAS,UAAU"}
|
package/dist/cli/index.js
CHANGED
package/dist/cli/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":[],"sources":["../../src/cli/index.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { Command } from \"commander\";\nimport { codemodCommand } from \"./commands/codemod/index.js\";\nimport { docsCommand } from \"./commands/docs.js\";\nimport { generateTypesCommand } from \"./commands/generate-types.js\";\nimport { lintCommand } from \"./commands/lint.js\";\nimport { pluginCommand } from \"./commands/plugin/index.js\";\nimport { setupCommand } from \"./commands/setup.js\";\n\nconst __dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgPath = join(__dirname, \"../../package.json\");\nconst pkg = JSON.parse(readFileSync(pkgPath, \"utf-8\"));\n\nconst cmd = new Command();\n\ncmd\n .name(\"appkit\")\n .description(\"CLI tools for Databricks AppKit\")\n .version(pkg.version);\n\ncmd.addCommand(setupCommand);\ncmd.addCommand(generateTypesCommand);\ncmd.addCommand(lintCommand);\ncmd.addCommand(docsCommand);\ncmd.addCommand(pluginCommand);\ncmd.addCommand(codemodCommand);\n\
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/cli/index.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\nimport { Command } from \"commander\";\nimport { codemodCommand } from \"./commands/codemod/index.js\";\nimport { docsCommand } from \"./commands/docs.js\";\nimport { generateTypesCommand } from \"./commands/generate-types.js\";\nimport { lintCommand } from \"./commands/lint.js\";\nimport { pluginCommand } from \"./commands/plugin/index.js\";\nimport { setupCommand } from \"./commands/setup.js\";\n\nconst __dirname = dirname(fileURLToPath(import.meta.url));\nconst pkgPath = join(__dirname, \"../../package.json\");\nconst pkg = JSON.parse(readFileSync(pkgPath, \"utf-8\"));\n\nconst cmd = new Command();\n\ncmd\n .name(\"appkit\")\n .description(\"CLI tools for Databricks AppKit\")\n .version(pkg.version);\n\ncmd.addCommand(setupCommand);\ncmd.addCommand(generateTypesCommand);\ncmd.addCommand(lintCommand);\ncmd.addCommand(docsCommand);\ncmd.addCommand(pluginCommand);\ncmd.addCommand(codemodCommand);\n\nawait cmd.parseAsync();\n"],"mappings":";;;;;;;;;;;;;;AAcA,MAAM,UAAU,KADE,QAAQ,cAAc,OAAO,KAAK,IAAI,CAAC,EACzB,qBAAqB;AACrD,MAAM,MAAM,KAAK,MAAM,aAAa,SAAS,QAAQ,CAAC;AAEtD,MAAM,MAAM,IAAI,SAAS;AAEzB,IACG,KAAK,SAAS,CACd,YAAY,kCAAkC,CAC9C,QAAQ,IAAI,QAAQ;AAEvB,IAAI,WAAW,aAAa;AAC5B,IAAI,WAAW,qBAAqB;AACpC,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,YAAY;AAC3B,IAAI,WAAW,cAAc;AAC7B,IAAI,WAAW,eAAe;AAE9B,MAAM,IAAI,YAAY"}
|
package/dist/connectors/index.js
CHANGED
|
@@ -11,7 +11,7 @@ import "./jobs/index.js";
|
|
|
11
11
|
import { buildMcpHostPolicy } from "./mcp/host-policy.js";
|
|
12
12
|
import { AppKitMcpClient } from "./mcp/client.js";
|
|
13
13
|
import "./mcp/index.js";
|
|
14
|
-
import { SQLWarehouseConnector } from "./sql-warehouse/client.js";
|
|
14
|
+
import { DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS, SQLWarehouseConnector } from "./sql-warehouse/client.js";
|
|
15
15
|
import "./sql-warehouse/index.js";
|
|
16
16
|
import "./vector-search/index.js";
|
|
17
17
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createLogger } from "../../logging/logger.js";
|
|
2
2
|
import { AppKitError } from "../../errors/base.js";
|
|
3
|
+
import { ConfigurationError } from "../../errors/configuration.js";
|
|
3
4
|
import { ConnectionError } from "../../errors/connection.js";
|
|
4
5
|
import { ExecutionError } from "../../errors/execution.js";
|
|
5
6
|
import { ValidationError } from "../../errors/validation.js";
|
|
@@ -8,14 +9,38 @@ import { TelemetryManager } from "../../telemetry/telemetry-manager.js";
|
|
|
8
9
|
import { SpanKind, SpanStatusCode } from "../../telemetry/index.js";
|
|
9
10
|
import { ArrowStreamProcessor } from "../../stream/arrow-stream-processor.js";
|
|
10
11
|
import { executeStatementDefaults } from "./defaults.js";
|
|
12
|
+
import { WarehousePollBackoff } from "./warehouse-poll-backoff.js";
|
|
13
|
+
import { WarehouseStatusEmitter } from "./warehouse-status-emitter.js";
|
|
11
14
|
import { Context } from "@databricks/sdk-experimental";
|
|
12
15
|
|
|
13
16
|
//#region src/connectors/sql-warehouse/client.ts
|
|
14
17
|
const logger = createLogger("connectors:sql-warehouse");
|
|
18
|
+
/**
|
|
19
|
+
* Default ceiling for how long {@link SQLWarehouseConnector.ensureWarehouseRunning}
|
|
20
|
+
* will wait for a warehouse to reach the RUNNING state before giving up.
|
|
21
|
+
*
|
|
22
|
+
* Five minutes covers a cold-start of a classic warehouse on most workspaces;
|
|
23
|
+
* serverless typically reaches RUNNING within ~30s.
|
|
24
|
+
*/
|
|
25
|
+
const DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS = 300 * 1e3;
|
|
26
|
+
/**
|
|
27
|
+
* Window during which a recent `RUNNING` observation lets subsequent calls
|
|
28
|
+
* to {@link SQLWarehouseConnector.ensureWarehouseRunning} short-circuit
|
|
29
|
+
* without making any SDK calls. Sized to roughly the upper bound of how
|
|
30
|
+
* long Databricks keeps a warehouse "stickily" available between requests
|
|
31
|
+
* — past that, we re-verify.
|
|
32
|
+
*/
|
|
33
|
+
const WAREHOUSE_RUNNING_CACHE_TTL_MS = 3e4;
|
|
15
34
|
var SQLWarehouseConnector = class {
|
|
16
35
|
name = "sql-warehouse";
|
|
17
36
|
config;
|
|
18
37
|
_arrowProcessor = null;
|
|
38
|
+
/**
|
|
39
|
+
* Per-warehouse cache of the last RUNNING observation timestamp. Used by
|
|
40
|
+
* {@link ensureWarehouseRunning} to short-circuit warm-path callers; see
|
|
41
|
+
* {@link WAREHOUSE_RUNNING_CACHE_TTL_MS}.
|
|
42
|
+
*/
|
|
43
|
+
_recentlyRunning = /* @__PURE__ */ new Map();
|
|
19
44
|
telemetry;
|
|
20
45
|
telemetryMetrics;
|
|
21
46
|
constructor(config) {
|
|
@@ -157,6 +182,148 @@ var SQLWarehouseConnector = class {
|
|
|
157
182
|
includePrefix: true
|
|
158
183
|
});
|
|
159
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* Wait until the SQL warehouse is in the `RUNNING` state, auto-starting it
|
|
187
|
+
* if currently `STOPPED`. Emits a {@link WarehouseStatusUpdate} whenever
|
|
188
|
+
* the observed state changes, so callers can surface progress (e.g. over
|
|
189
|
+
* SSE) instead of letting the UI freeze on a cold warehouse. Equal
|
|
190
|
+
* successive observations are de-duplicated.
|
|
191
|
+
*
|
|
192
|
+
* Fast path: if this connector recently observed the warehouse RUNNING
|
|
193
|
+
* (within {@link WAREHOUSE_RUNNING_CACHE_TTL_MS}), the call returns
|
|
194
|
+
* immediately without any SDK round-trip or status emission. This keeps
|
|
195
|
+
* cache-hit analytics requests off the Databricks control plane.
|
|
196
|
+
*
|
|
197
|
+
* Behaviour by initial state:
|
|
198
|
+
* - `RUNNING`: emits one update, caches the observation, returns.
|
|
199
|
+
* - `STOPPED`: emits a synthetic `STARTING` update, calls
|
|
200
|
+
* `workspaceClient.warehouses.start`, then polls until `RUNNING`.
|
|
201
|
+
* When `autoStart: false`, throws `ConfigurationError` instead.
|
|
202
|
+
* - `STARTING` / `STOPPING`: polls until `RUNNING`.
|
|
203
|
+
* - `DELETED` / `DELETING`: throws `ConfigurationError.resourceNotFound`.
|
|
204
|
+
*
|
|
205
|
+
* Aborts and timeouts surface as `ExecutionError`. The poll loop uses
|
|
206
|
+
* exponential backoff with ±15% jitter (1s → 30s cap) and runs inside a
|
|
207
|
+
* `sql.warehouseReady` telemetry span.
|
|
208
|
+
*/
|
|
209
|
+
async ensureWarehouseRunning(workspaceClient, warehouseId, opts) {
|
|
210
|
+
const { onStatus, signal, timeoutMs = DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS, autoStart = true } = opts;
|
|
211
|
+
if (signal?.aborted) throw ExecutionError.canceled();
|
|
212
|
+
if (!warehouseId) throw ValidationError.missingField("warehouse_id");
|
|
213
|
+
if (this._isRecentlyRunning(warehouseId)) return;
|
|
214
|
+
return this.telemetry.startActiveSpan("sql.warehouseReady", {
|
|
215
|
+
kind: SpanKind.CLIENT,
|
|
216
|
+
attributes: {
|
|
217
|
+
"db.system": "databricks",
|
|
218
|
+
"db.warehouse_id": warehouseId,
|
|
219
|
+
"db.warehouse.startup_timeout_ms": timeoutMs
|
|
220
|
+
}
|
|
221
|
+
}, async (span) => {
|
|
222
|
+
const startTime = Date.now();
|
|
223
|
+
const emitter = new WarehouseStatusEmitter(span, startTime, onStatus);
|
|
224
|
+
let didStart = false;
|
|
225
|
+
const backoff = new WarehousePollBackoff();
|
|
226
|
+
try {
|
|
227
|
+
while (true) {
|
|
228
|
+
if (signal?.aborted) throw ExecutionError.canceled();
|
|
229
|
+
if (Date.now() - startTime > timeoutMs) throw ExecutionError.statementFailed(`SQL warehouse did not reach RUNNING within ${timeoutMs}ms`);
|
|
230
|
+
const info = await workspaceClient.warehouses.get({ id: warehouseId }, this._createContext(signal));
|
|
231
|
+
const state = info?.state;
|
|
232
|
+
const summary = info?.health?.summary;
|
|
233
|
+
switch (state) {
|
|
234
|
+
case "RUNNING":
|
|
235
|
+
emitter.emit(state, summary);
|
|
236
|
+
this._recentlyRunning.set(warehouseId, Date.now());
|
|
237
|
+
span.setAttribute("db.warehouse.attempts", emitter.attempt);
|
|
238
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
239
|
+
return;
|
|
240
|
+
case "DELETED":
|
|
241
|
+
case "DELETING": throw ConfigurationError.resourceNotFound("Warehouse ID", `The configured SQL warehouse is ${state}. Update DATABRICKS_WAREHOUSE_ID to point at an active warehouse.`);
|
|
242
|
+
case "STOPPED":
|
|
243
|
+
if (!autoStart) throw new ConfigurationError("The configured SQL warehouse is STOPPED and analytics auto-start is disabled. Start the warehouse manually or set analytics.autoStartWarehouse=true.");
|
|
244
|
+
if (!didStart) {
|
|
245
|
+
emitter.emit("STARTING", summary);
|
|
246
|
+
await workspaceClient.warehouses.start({ id: warehouseId }, this._createContext(signal));
|
|
247
|
+
didStart = true;
|
|
248
|
+
} else emitter.emit(state, summary);
|
|
249
|
+
break;
|
|
250
|
+
case "STARTING":
|
|
251
|
+
case "STOPPING":
|
|
252
|
+
emitter.emit(state, summary);
|
|
253
|
+
break;
|
|
254
|
+
default: throw ExecutionError.unknownState(String(state ?? "unknown"));
|
|
255
|
+
}
|
|
256
|
+
const sleepMs = backoff.next();
|
|
257
|
+
if (Date.now() + sleepMs - startTime >= timeoutMs) throw ExecutionError.statementFailed(`SQL warehouse did not reach RUNNING within ${timeoutMs}ms`);
|
|
258
|
+
await this._sleepRespectingAbort(sleepMs, signal);
|
|
259
|
+
}
|
|
260
|
+
} catch (error) {
|
|
261
|
+
this._throwSanitizedReadinessError(error, warehouseId, span);
|
|
262
|
+
} finally {
|
|
263
|
+
span.end();
|
|
264
|
+
}
|
|
265
|
+
}, {
|
|
266
|
+
name: this.name,
|
|
267
|
+
includePrefix: true
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* `true` if this connector observed `warehouseId` in the RUNNING state
|
|
272
|
+
* within the recent-cache TTL.
|
|
273
|
+
*/
|
|
274
|
+
_isRecentlyRunning(warehouseId) {
|
|
275
|
+
const observedAt = this._recentlyRunning.get(warehouseId);
|
|
276
|
+
return observedAt !== void 0 && Date.now() - observedAt < WAREHOUSE_RUNNING_CACHE_TTL_MS;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Final landing for any error thrown by the readiness poll loop. Records
|
|
280
|
+
* the original on the span and (at debug level) on the logger, then
|
|
281
|
+
* rethrows either the structured AppKitError unchanged or a curated
|
|
282
|
+
* ExecutionError — never the raw SDK message, which can contain operator
|
|
283
|
+
* internals.
|
|
284
|
+
*
|
|
285
|
+
* The `logger.debug` covers environments where OTel isn't configured
|
|
286
|
+
* (common in dev) so the raw error isn't lost just because no span
|
|
287
|
+
* exporter is hooked up.
|
|
288
|
+
*/
|
|
289
|
+
_throwSanitizedReadinessError(error, warehouseId, span) {
|
|
290
|
+
if (error instanceof AppKitError) {
|
|
291
|
+
span.recordException(error);
|
|
292
|
+
span.setStatus({
|
|
293
|
+
code: SpanStatusCode.ERROR,
|
|
294
|
+
message: error.code
|
|
295
|
+
});
|
|
296
|
+
throw error;
|
|
297
|
+
}
|
|
298
|
+
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
299
|
+
logger.debug("Warehouse readiness check raw error for %s: %O", warehouseId, error);
|
|
300
|
+
const wrapped = ExecutionError.statementFailed("Warehouse readiness check failed");
|
|
301
|
+
span.setStatus({
|
|
302
|
+
code: SpanStatusCode.ERROR,
|
|
303
|
+
message: wrapped.code
|
|
304
|
+
});
|
|
305
|
+
throw wrapped;
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Sleep for `ms` milliseconds, but resolve early (and reject with
|
|
309
|
+
* `ExecutionError.canceled()`) if `signal` aborts mid-sleep. Used by the
|
|
310
|
+
* warehouse readiness loop so that client disconnects don't keep the loop
|
|
311
|
+
* polling for the full interval.
|
|
312
|
+
*/
|
|
313
|
+
_sleepRespectingAbort(ms, signal) {
|
|
314
|
+
if (signal?.aborted) return Promise.reject(ExecutionError.canceled());
|
|
315
|
+
return new Promise((resolve, reject) => {
|
|
316
|
+
const timer = setTimeout(() => {
|
|
317
|
+
signal?.removeEventListener("abort", onAbort);
|
|
318
|
+
resolve();
|
|
319
|
+
}, ms);
|
|
320
|
+
const onAbort = () => {
|
|
321
|
+
clearTimeout(timer);
|
|
322
|
+
reject(ExecutionError.canceled());
|
|
323
|
+
};
|
|
324
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
325
|
+
});
|
|
326
|
+
}
|
|
160
327
|
async _pollForStatementResult(workspaceClient, statementId, timeout = executeStatementDefaults.timeout, signal) {
|
|
161
328
|
return this.telemetry.startActiveSpan("sql.poll", { attributes: {
|
|
162
329
|
"db.statement_id": statementId,
|
|
@@ -323,5 +490,5 @@ var SQLWarehouseConnector = class {
|
|
|
323
490
|
};
|
|
324
491
|
|
|
325
492
|
//#endregion
|
|
326
|
-
export { SQLWarehouseConnector };
|
|
493
|
+
export { DEFAULT_WAREHOUSE_STARTUP_TIMEOUT_MS, SQLWarehouseConnector };
|
|
327
494
|
//# sourceMappingURL=client.js.map
|