@databricks/appkit 0.41.4 → 0.41.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/appkit/package.js +1 -1
- package/dist/shared/src/schemas/metric-fqn.js +85 -0
- package/dist/shared/src/schemas/metric-fqn.js.map +1 -0
- package/dist/type-generator/cache.js +33 -1
- package/dist/type-generator/cache.js.map +1 -1
- package/dist/type-generator/errors.js +100 -0
- package/dist/type-generator/errors.js.map +1 -0
- package/dist/type-generator/index.js +190 -2
- package/dist/type-generator/index.js.map +1 -1
- package/dist/type-generator/mv-registry/config.js +131 -0
- package/dist/type-generator/mv-registry/config.js.map +1 -0
- package/dist/type-generator/mv-registry/describe.js +230 -0
- package/dist/type-generator/mv-registry/describe.js.map +1 -0
- package/dist/type-generator/mv-registry/metadata.js +40 -0
- package/dist/type-generator/mv-registry/metadata.js.map +1 -0
- package/dist/type-generator/mv-registry/render-types.js +118 -0
- package/dist/type-generator/mv-registry/render-types.js.map +1 -0
- package/dist/type-generator/mv-registry/sync.js +96 -0
- package/dist/type-generator/mv-registry/sync.js.map +1 -0
- package/dist/type-generator/query-registry.js +126 -88
- package/dist/type-generator/query-registry.js.map +1 -1
- package/dist/type-generator/statement-result.js +116 -0
- package/dist/type-generator/statement-result.js.map +1 -0
- package/dist/type-generator/types.js.map +1 -1
- package/dist/type-generator/vite-plugin.d.ts +12 -0
- package/dist/type-generator/vite-plugin.d.ts.map +1 -1
- package/dist/type-generator/vite-plugin.js +11 -5
- package/dist/type-generator/vite-plugin.js.map +1 -1
- package/docs/development/type-generation.md +4 -0
- package/docs/plugins/analytics.md +15 -0
- package/package.json +2 -1
- package/sbom.cdx.json +1 -1
package/dist/appkit/package.js
CHANGED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
//#region ../shared/src/schemas/metric-fqn.ts
|
|
2
|
+
/**
|
|
3
|
+
* Unity Catalog object-name grammar - the single source of truth for metric
|
|
4
|
+
* view FQN naming validation.
|
|
5
|
+
*
|
|
6
|
+
* This module is deliberately **zod-free**. A metric view's `source` FQN is
|
|
7
|
+
* validated in two places that must agree:
|
|
8
|
+
*
|
|
9
|
+
* 1. The canonical Zod schema (`./metric-source.ts`), which composes the
|
|
10
|
+
* three-part FQN regex from {@link UC_FQN_PATTERN} for IDE/CI and the
|
|
11
|
+
* generated JSON schema (`docs/static/schemas/metric-source.schema.json`).
|
|
12
|
+
* 2. The type-generator runtime (`packages/appkit/src/type-generator/mv-registry/config.ts`),
|
|
13
|
+
* which imports {@link UC_FQN_PATTERN} as a plain value to validate each
|
|
14
|
+
* dot-split segment.
|
|
15
|
+
*
|
|
16
|
+
* The type-generator's runtime path must NOT pull the shared Zod schema package
|
|
17
|
+
* in (locked dependency-graph ruling - see the comment in
|
|
18
|
+
* `packages/appkit/src/type-generator/cache.ts`). Keeping the pattern in this
|
|
19
|
+
* zod-free module lets the runtime import the regex without dragging zod into
|
|
20
|
+
* its bundle, while still single-sourcing the grammar.
|
|
21
|
+
*
|
|
22
|
+
* -- UC delimited (quoted) object-name rules ------------------------------
|
|
23
|
+
* The metric view FQN is always backtick-quoted before interpolation into SQL
|
|
24
|
+
* (see `quoteFqnForSql` in the type-generator), so the **delimited identifier**
|
|
25
|
+
* grammar is the one that applies - not the narrower unquoted-identifier rule.
|
|
26
|
+
*
|
|
27
|
+
* Per the Databricks SQL names reference, a Unity Catalog object name:
|
|
28
|
+
* - cannot exceed 255 characters ({@link MAX_UC_OBJECT_NAME_LENGTH}); and
|
|
29
|
+
* - cannot contain any of these characters:
|
|
30
|
+
* - period (`.`)
|
|
31
|
+
* - space (U+0020)
|
|
32
|
+
* - forward slash (`/`)
|
|
33
|
+
* - all ASCII control characters (U+0000-U+001F)
|
|
34
|
+
* - the DELETE character (U+007F)
|
|
35
|
+
*
|
|
36
|
+
* Every other character is permitted in a quoted name, including non-ASCII
|
|
37
|
+
* letters (the docs demonstrate Chinese/Russian/Portuguese names) and hyphens.
|
|
38
|
+
* This is intentionally broader than the old hand-rolled allowlist
|
|
39
|
+
* (`[a-zA-Z0-9_-]`), which was flagged in PR #433 review (pkosiec: "more
|
|
40
|
+
* restrictive than UC"): the goal is to accept what UC accepts as a quoted
|
|
41
|
+
* name and reject only what UC rejects.
|
|
42
|
+
*
|
|
43
|
+
* Verified against the Databricks docs on 2026-06-19:
|
|
44
|
+
* https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-names
|
|
45
|
+
* (the link cited in the PR #433 review). If the published rules change,
|
|
46
|
+
* re-confirm against that page.
|
|
47
|
+
*
|
|
48
|
+
* @note The period is excluded here because it is the FQN segment separator -
|
|
49
|
+
* a name containing a literal dot cannot be expressed in the dotted `source`
|
|
50
|
+
* string at all. The dotted-source arity (exactly three segments) and the
|
|
51
|
+
* 255-char-per-segment cap are enforced structurally by the callers; this
|
|
52
|
+
* pattern only encodes the per-segment allowed character set.
|
|
53
|
+
*/
|
|
54
|
+
/**
|
|
55
|
+
* Maximum length, in characters, of a single Unity Catalog object name
|
|
56
|
+
* (catalog, schema, or metric view). UC rejects names longer than this.
|
|
57
|
+
*/
|
|
58
|
+
const MAX_UC_OBJECT_NAME_LENGTH = 255;
|
|
59
|
+
/**
|
|
60
|
+
* Matches a single, non-empty Unity Catalog object name as it may appear in a
|
|
61
|
+
* backtick-quoted (delimited) identifier - one segment of a metric view FQN.
|
|
62
|
+
*
|
|
63
|
+
* Accepts any non-empty run of characters EXCEPT the UC-prohibited set:
|
|
64
|
+
* period, space, forward slash, ASCII control characters (U+0000-U+001F), and
|
|
65
|
+
* DELETE (U+007F). Length is NOT bounded here - callers enforce
|
|
66
|
+
* {@link MAX_UC_OBJECT_NAME_LENGTH} separately so they can emit a precise
|
|
67
|
+
* "segment too long" message distinct from a charset violation.
|
|
68
|
+
*
|
|
69
|
+
* The negated character class encodes the prohibited set as one contiguous
|
|
70
|
+
* range plus singletons: U+0000-U+0020 (every ASCII control character plus the
|
|
71
|
+
* space, which sits at U+0020 immediately after the control range), U+007F
|
|
72
|
+
* (DELETE), `.` (period - also the FQN segment separator), and `/` (slash).
|
|
73
|
+
*
|
|
74
|
+
* @example
|
|
75
|
+
* UC_FQN_PATTERN.test("revenue_metrics"); // true
|
|
76
|
+
* UC_FQN_PATTERN.test("prod-data"); // true (hyphens are UC-legal)
|
|
77
|
+
* UC_FQN_PATTERN.test("cafe\u0301"); // true (non-ASCII is UC-legal)
|
|
78
|
+
* UC_FQN_PATTERN.test("bad name"); // false (space is prohibited)
|
|
79
|
+
* UC_FQN_PATTERN.test("a/b"); // false (slash is prohibited)
|
|
80
|
+
*/
|
|
81
|
+
const UC_FQN_PATTERN = /^[^\x00-\x20\x7f./]+$/;
|
|
82
|
+
|
|
83
|
+
//#endregion
|
|
84
|
+
export { MAX_UC_OBJECT_NAME_LENGTH, UC_FQN_PATTERN };
|
|
85
|
+
//# sourceMappingURL=metric-fqn.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metric-fqn.js","names":[],"sources":["../../../../../shared/src/schemas/metric-fqn.ts"],"sourcesContent":["/**\n * Unity Catalog object-name grammar - the single source of truth for metric\n * view FQN naming validation.\n *\n * This module is deliberately **zod-free**. A metric view's `source` FQN is\n * validated in two places that must agree:\n *\n * 1. The canonical Zod schema (`./metric-source.ts`), which composes the\n * three-part FQN regex from {@link UC_FQN_PATTERN} for IDE/CI and the\n * generated JSON schema (`docs/static/schemas/metric-source.schema.json`).\n * 2. The type-generator runtime (`packages/appkit/src/type-generator/mv-registry/config.ts`),\n * which imports {@link UC_FQN_PATTERN} as a plain value to validate each\n * dot-split segment.\n *\n * The type-generator's runtime path must NOT pull the shared Zod schema package\n * in (locked dependency-graph ruling - see the comment in\n * `packages/appkit/src/type-generator/cache.ts`). Keeping the pattern in this\n * zod-free module lets the runtime import the regex without dragging zod into\n * its bundle, while still single-sourcing the grammar.\n *\n * -- UC delimited (quoted) object-name rules ------------------------------\n * The metric view FQN is always backtick-quoted before interpolation into SQL\n * (see `quoteFqnForSql` in the type-generator), so the **delimited identifier**\n * grammar is the one that applies - not the narrower unquoted-identifier rule.\n *\n * Per the Databricks SQL names reference, a Unity Catalog object name:\n * - cannot exceed 255 characters ({@link MAX_UC_OBJECT_NAME_LENGTH}); and\n * - cannot contain any of these characters:\n * - period (`.`)\n * - space (U+0020)\n * - forward slash (`/`)\n * - all ASCII control characters (U+0000-U+001F)\n * - the DELETE character (U+007F)\n *\n * Every other character is permitted in a quoted name, including non-ASCII\n * letters (the docs demonstrate Chinese/Russian/Portuguese names) and hyphens.\n * This is intentionally broader than the old hand-rolled allowlist\n * (`[a-zA-Z0-9_-]`), which was flagged in PR #433 review (pkosiec: \"more\n * restrictive than UC\"): the goal is to accept what UC accepts as a quoted\n * name and reject only what UC rejects.\n *\n * Verified against the Databricks docs on 2026-06-19:\n * https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-names\n * (the link cited in the PR #433 review). If the published rules change,\n * re-confirm against that page.\n *\n * @note The period is excluded here because it is the FQN segment separator -\n * a name containing a literal dot cannot be expressed in the dotted `source`\n * string at all. The dotted-source arity (exactly three segments) and the\n * 255-char-per-segment cap are enforced structurally by the callers; this\n * pattern only encodes the per-segment allowed character set.\n */\n\n/**\n * Maximum length, in characters, of a single Unity Catalog object name\n * (catalog, schema, or metric view). UC rejects names longer than this.\n */\nexport const MAX_UC_OBJECT_NAME_LENGTH = 255;\n\n/**\n * Matches a single, non-empty Unity Catalog object name as it may appear in a\n * backtick-quoted (delimited) identifier - one segment of a metric view FQN.\n *\n * Accepts any non-empty run of characters EXCEPT the UC-prohibited set:\n * period, space, forward slash, ASCII control characters (U+0000-U+001F), and\n * DELETE (U+007F). Length is NOT bounded here - callers enforce\n * {@link MAX_UC_OBJECT_NAME_LENGTH} separately so they can emit a precise\n * \"segment too long\" message distinct from a charset violation.\n *\n * The negated character class encodes the prohibited set as one contiguous\n * range plus singletons: U+0000-U+0020 (every ASCII control character plus the\n * space, which sits at U+0020 immediately after the control range), U+007F\n * (DELETE), `.` (period - also the FQN segment separator), and `/` (slash).\n *\n * @example\n * UC_FQN_PATTERN.test(\"revenue_metrics\"); // true\n * UC_FQN_PATTERN.test(\"prod-data\"); // true (hyphens are UC-legal)\n * UC_FQN_PATTERN.test(\"cafe\\u0301\"); // true (non-ASCII is UC-legal)\n * UC_FQN_PATTERN.test(\"bad name\"); // false (space is prohibited)\n * UC_FQN_PATTERN.test(\"a/b\"); // false (slash is prohibited)\n */\n// biome-ignore lint/suspicious/noControlCharactersInRegex: UC explicitly prohibits ASCII control characters in object names; this negated class encodes that rule.\nexport const UC_FQN_PATTERN = /^[^\\x00-\\x20\\x7f./]+$/;\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyDA,MAAa,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;AAyBzC,MAAa,iBAAiB"}
|
|
@@ -5,6 +5,30 @@ import path from "node:path";
|
|
|
5
5
|
|
|
6
6
|
//#region src/type-generator/cache.ts
|
|
7
7
|
const logger = createLogger("type-generator:cache");
|
|
8
|
+
/**
|
|
9
|
+
* Structural gate for reviving a cached metric entry at partition time.
|
|
10
|
+
*
|
|
11
|
+
* The cache file lives in `node_modules/.databricks` and is plain JSON —
|
|
12
|
+
* hand-edits, truncation, or a stale writer can leave entries whose shape no
|
|
13
|
+
* longer matches {@link MetricCacheEntry}. A malformed entry must read as a
|
|
14
|
+
* cache MISS (re-describe) rather than crash the pass or render revived
|
|
15
|
+
* garbage into the artifacts. Checks exactly what the renderers and the
|
|
16
|
+
* metadata bundle consume: `hash` string, `retry` boolean, and a schema with
|
|
17
|
+
* `key`/`source` strings, a valid lane, an optional boolean `degraded`, and
|
|
18
|
+
* measure/dimension arrays whose elements carry `name`/`type` strings
|
|
19
|
+
* (other column fields are optional). Deliberately inline — the shared Zod
|
|
20
|
+
* schemas must not enter the type-generator's runtime path.
|
|
21
|
+
*/
|
|
22
|
+
function isRevivableMetricCacheEntry(entry) {
|
|
23
|
+
if (typeof entry !== "object" || entry === null) return false;
|
|
24
|
+
const e = entry;
|
|
25
|
+
if (typeof e.hash !== "string" || typeof e.retry !== "boolean") return false;
|
|
26
|
+
const schema = e.schema;
|
|
27
|
+
if (typeof schema !== "object" || schema === null || Array.isArray(schema)) return false;
|
|
28
|
+
const s = schema;
|
|
29
|
+
const isColumnArray = (value) => Array.isArray(value) && value.every((col) => typeof col === "object" && col !== null && typeof col.name === "string" && typeof col.type === "string");
|
|
30
|
+
return typeof s.key === "string" && typeof s.source === "string" && (s.lane === "sp" || s.lane === "obo") && (s.degraded === void 0 || typeof s.degraded === "boolean") && isColumnArray(s.measures) && isColumnArray(s.dimensions);
|
|
31
|
+
}
|
|
8
32
|
const CACHE_VERSION = "3";
|
|
9
33
|
const CACHE_FILE = ".appkit-types-cache.json";
|
|
10
34
|
const CACHE_DIR = path.join(process.cwd(), "node_modules", ".databricks", "appkit");
|
|
@@ -18,6 +42,14 @@ function hashSQL(sql) {
|
|
|
18
42
|
return crypto.createHash("md5").update(sql).digest("hex");
|
|
19
43
|
}
|
|
20
44
|
/**
|
|
45
|
+
* Change detector stored on {@link MetricCacheEntry.hash}: md5 over
|
|
46
|
+
* `"<source>|<lane>"` — the two config inputs that determine a DESCRIBE —
|
|
47
|
+
* so editing either invalidates the entry.
|
|
48
|
+
*/
|
|
49
|
+
function metricCacheHash(source, lane) {
|
|
50
|
+
return hashSQL(`${source}|${lane}`);
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
21
53
|
* Load the cache from the file system
|
|
22
54
|
* If the cache is not found, run the query explain
|
|
23
55
|
* @returns - the cache
|
|
@@ -47,5 +79,5 @@ async function saveCache(cache) {
|
|
|
47
79
|
}
|
|
48
80
|
|
|
49
81
|
//#endregion
|
|
50
|
-
export { CACHE_VERSION, hashSQL, loadCache, saveCache };
|
|
82
|
+
export { CACHE_VERSION, hashSQL, isRevivableMetricCacheEntry, loadCache, metricCacheHash, saveCache };
|
|
51
83
|
//# sourceMappingURL=cache.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","names":[],"sources":["../../src/type-generator/cache.ts"],"sourcesContent":["import crypto from \"node:crypto\";\nimport fs from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { createLogger } from \"../logging/logger\";\n\nconst logger = createLogger(\"type-generator:cache\");\n\n/**\n * Cache types\n * @property hash - the hash of the SQL query\n * @property type - the type of the query\n */\ninterface CacheEntry {\n hash: string;\n type: string;\n retry: boolean;\n}\n\n/**\n * Cache interface\n * @property version - the version of the cache\n * @property queries - the queries in the cache\n */\ninterface Cache {\n version: string;\n queries: Record<string, CacheEntry>;\n}\n\nexport const CACHE_VERSION = \"3\";\nconst CACHE_FILE = \".appkit-types-cache.json\";\nconst CACHE_DIR = path.join(\n process.cwd(),\n \"node_modules\",\n \".databricks\",\n \"appkit\",\n);\n\n/**\n * Hash the SQL query\n * Uses MD5 to hash the SQL query\n * @param sql - the SQL query to hash\n * @returns - the hash of the SQL query\n */\nexport function hashSQL(sql: string): string {\n return crypto.createHash(\"md5\").update(sql).digest(\"hex\");\n}\n\n/**\n * Load the cache from the file system\n * If the cache is not found, run the query explain\n * @returns - the cache\n */\nexport async function loadCache(): Promise<Cache> {\n const cachePath = path.join(CACHE_DIR, CACHE_FILE);\n try {\n await fs.mkdir(CACHE_DIR, { recursive: true });\n\n const raw = await fs.readFile(cachePath, \"utf8\");\n const cache = JSON.parse(raw) as Cache;\n if (cache.version === CACHE_VERSION) {\n return cache;\n }\n } catch (err) {\n if ((err as NodeJS.ErrnoException).code !== \"ENOENT\") {\n logger.warn(\"Cache file is corrupted, flushing cache completely.\");\n }\n }\n return { version: CACHE_VERSION, queries: {} };\n}\n\n/**\n * Save the cache to the file system\n * @param cache - cache object to save\n */\nexport async function saveCache(cache: Cache): Promise<void> {\n const cachePath = path.join(CACHE_DIR, CACHE_FILE);\n await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), \"utf8\");\n}\n"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"cache.js","names":[],"sources":["../../src/type-generator/cache.ts"],"sourcesContent":["import crypto from \"node:crypto\";\nimport fs from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { createLogger } from \"../logging/logger\";\nimport type { MetricSchema } from \"./mv-registry/types\";\n\nconst logger = createLogger(\"type-generator:cache\");\n\n/**\n * Cache types\n * @property hash - the hash of the SQL query\n * @property type - the type of the query\n * @property retry - when true the entry never satisfies a cache hit, so the\n * query is re-described on the next pass; fresh successful describes\n * persist `retry: false`\n */\ninterface CacheEntry {\n hash: string;\n type: string;\n retry: boolean;\n}\n\n/**\n * One cached metric-view DESCRIBE outcome.\n *\n * `hash` is md5 over `\"<source>|<lane>\"` — the two config inputs that\n * determine a DESCRIBE — so editing either invalidates the entry. `schema`\n * is the full {@link MetricSchema} persisted verbatim (it is JSON-safe by\n * design), letting a warm pass regenerate both metric artifacts without a\n * single warehouse call. `retry: true` marks a SELF-CONVERGING degraded\n * outcome (DESCRIBE skipped behind a not-running warehouse, unanswered, or\n * transiently failed): the cached schema still renders artifacts, but the\n * next eligible pass re-describes exactly these keys so degraded schemas\n * converge to real ones. A degraded schema with `retry: false` is a STICKY\n * failure — a deterministic DESCRIBE failure (bad FQN, unparseable\n * response, zero columns) or a deleted warehouse — that re-describing the\n * unchanged entry cannot fix; it hits like any cached entry until the\n * config hash changes or the cache is bypassed, and the type generator\n * warns about it on every pass that serves it.\n */\nexport interface MetricCacheEntry {\n hash: string;\n schema: MetricSchema;\n retry: boolean;\n}\n\n/**\n * Structural gate for reviving a cached metric entry at partition time.\n *\n * The cache file lives in `node_modules/.databricks` and is plain JSON —\n * hand-edits, truncation, or a stale writer can leave entries whose shape no\n * longer matches {@link MetricCacheEntry}. A malformed entry must read as a\n * cache MISS (re-describe) rather than crash the pass or render revived\n * garbage into the artifacts. Checks exactly what the renderers and the\n * metadata bundle consume: `hash` string, `retry` boolean, and a schema with\n * `key`/`source` strings, a valid lane, an optional boolean `degraded`, and\n * measure/dimension arrays whose elements carry `name`/`type` strings\n * (other column fields are optional). Deliberately inline — the shared Zod\n * schemas must not enter the type-generator's runtime path.\n */\nexport function isRevivableMetricCacheEntry(entry: unknown): boolean {\n if (typeof entry !== \"object\" || entry === null) return false;\n const e = entry as Record<string, unknown>;\n if (typeof e.hash !== \"string\" || typeof e.retry !== \"boolean\") {\n return false;\n }\n const schema = e.schema;\n if (typeof schema !== \"object\" || schema === null || Array.isArray(schema)) {\n return false;\n }\n const s = schema as Record<string, unknown>;\n const isColumnArray = (value: unknown): boolean =>\n Array.isArray(value) &&\n value.every(\n (col) =>\n typeof col === \"object\" &&\n col !== null &&\n typeof (col as Record<string, unknown>).name === \"string\" &&\n typeof (col as Record<string, unknown>).type === \"string\",\n );\n return (\n typeof s.key === \"string\" &&\n typeof s.source === \"string\" &&\n (s.lane === \"sp\" || s.lane === \"obo\") &&\n (s.degraded === undefined || typeof s.degraded === \"boolean\") &&\n isColumnArray(s.measures) &&\n isColumnArray(s.dimensions)\n );\n}\n\n/**\n * Cache interface\n * @property version - the version of the cache\n * @property queries - the queries in the cache\n * @property metrics - cached metric-view schemas keyed by metric key.\n * OPTIONAL on purpose: version \"3\" files written before this section\n * existed load unchanged (absent ⇒ treated as empty by the metric path),\n * and the query path's `noCache` reinit literal stays valid as-is. The\n * section rides through the query path's load → mutate → save cycle as a\n * plain sibling key, so query-side saves preserve it byte-for-byte.\n */\ninterface Cache {\n version: string;\n queries: Record<string, CacheEntry>;\n metrics?: Record<string, MetricCacheEntry>;\n}\n\nexport const CACHE_VERSION = \"3\";\nconst CACHE_FILE = \".appkit-types-cache.json\";\nconst CACHE_DIR = path.join(\n process.cwd(),\n \"node_modules\",\n \".databricks\",\n \"appkit\",\n);\n\n/**\n * Hash the SQL query\n * Uses MD5 to hash the SQL query\n * @param sql - the SQL query to hash\n * @returns - the hash of the SQL query\n */\nexport function hashSQL(sql: string): string {\n return crypto.createHash(\"md5\").update(sql).digest(\"hex\");\n}\n\n/**\n * Change detector stored on {@link MetricCacheEntry.hash}: md5 over\n * `\"<source>|<lane>\"` — the two config inputs that determine a DESCRIBE —\n * so editing either invalidates the entry.\n */\nexport function metricCacheHash(source: string, lane: string): string {\n return hashSQL(`${source}|${lane}`);\n}\n\n/**\n * Load the cache from the file system\n * If the cache is not found, run the query explain\n * @returns - the cache\n */\nexport async function loadCache(): Promise<Cache> {\n const cachePath = path.join(CACHE_DIR, CACHE_FILE);\n try {\n await fs.mkdir(CACHE_DIR, { recursive: true });\n\n const raw = await fs.readFile(cachePath, \"utf8\");\n const cache = JSON.parse(raw) as Cache;\n if (cache.version === CACHE_VERSION) {\n return cache;\n }\n } catch (err) {\n if ((err as NodeJS.ErrnoException).code !== \"ENOENT\") {\n logger.warn(\"Cache file is corrupted, flushing cache completely.\");\n }\n }\n return { version: CACHE_VERSION, queries: {} };\n}\n\n/**\n * Save the cache to the file system\n * @param cache - cache object to save\n */\nexport async function saveCache(cache: Cache): Promise<void> {\n const cachePath = path.join(CACHE_DIR, CACHE_FILE);\n await fs.writeFile(cachePath, JSON.stringify(cache, null, 2), \"utf8\");\n}\n"],"mappings":";;;;;;AAMA,MAAM,SAAS,aAAa,uBAAuB;;;;;;;;;;;;;;;AAsDnD,SAAgB,4BAA4B,OAAyB;AACnE,KAAI,OAAO,UAAU,YAAY,UAAU,KAAM,QAAO;CACxD,MAAM,IAAI;AACV,KAAI,OAAO,EAAE,SAAS,YAAY,OAAO,EAAE,UAAU,UACnD,QAAO;CAET,MAAM,SAAS,EAAE;AACjB,KAAI,OAAO,WAAW,YAAY,WAAW,QAAQ,MAAM,QAAQ,OAAO,CACxE,QAAO;CAET,MAAM,IAAI;CACV,MAAM,iBAAiB,UACrB,MAAM,QAAQ,MAAM,IACpB,MAAM,OACH,QACC,OAAO,QAAQ,YACf,QAAQ,QACR,OAAQ,IAAgC,SAAS,YACjD,OAAQ,IAAgC,SAAS,SACpD;AACH,QACE,OAAO,EAAE,QAAQ,YACjB,OAAO,EAAE,WAAW,aACnB,EAAE,SAAS,QAAQ,EAAE,SAAS,WAC9B,EAAE,aAAa,UAAa,OAAO,EAAE,aAAa,cACnD,cAAc,EAAE,SAAS,IACzB,cAAc,EAAE,WAAW;;AAqB/B,MAAa,gBAAgB;AAC7B,MAAM,aAAa;AACnB,MAAM,YAAY,KAAK,KACrB,QAAQ,KAAK,EACb,gBACA,eACA,SACD;;;;;;;AAQD,SAAgB,QAAQ,KAAqB;AAC3C,QAAO,OAAO,WAAW,MAAM,CAAC,OAAO,IAAI,CAAC,OAAO,MAAM;;;;;;;AAQ3D,SAAgB,gBAAgB,QAAgB,MAAsB;AACpE,QAAO,QAAQ,GAAG,OAAO,GAAG,OAAO;;;;;;;AAQrC,eAAsB,YAA4B;CAChD,MAAM,YAAY,KAAK,KAAK,WAAW,WAAW;AAClD,KAAI;AACF,QAAM,GAAG,MAAM,WAAW,EAAE,WAAW,MAAM,CAAC;EAE9C,MAAM,MAAM,MAAM,GAAG,SAAS,WAAW,OAAO;EAChD,MAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,MAAI,MAAM,YAAY,cACpB,QAAO;UAEF,KAAK;AACZ,MAAK,IAA8B,SAAS,SAC1C,QAAO,KAAK,sDAAsD;;AAGtE,QAAO;EAAE,SAAS;EAAe,SAAS,EAAE;EAAE;;;;;;AAOhD,eAAsB,UAAU,OAA6B;CAC3D,MAAM,YAAY,KAAK,KAAK,WAAW,WAAW;AAClD,OAAM,GAAG,UAAU,WAAW,KAAK,UAAU,OAAO,MAAM,EAAE,EAAE,OAAO"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
//#region src/type-generator/errors.ts
|
|
2
|
+
/**
|
|
3
|
+
* Shared error-introspection utilities for the type generator.
|
|
4
|
+
*
|
|
5
|
+
* Both describe paths — the query registry and the metric-view registry — must
|
|
6
|
+
* classify a thrown failure the same way: a transient connectivity blip (which
|
|
7
|
+
* self-converges and should never fail a build) versus a deterministic failure
|
|
8
|
+
* (auth, bad warehouse id, truncated result, malformed request) that must be
|
|
9
|
+
* surfaced. Keeping `isConnectivityError` (and the message/diagnostic helpers it
|
|
10
|
+
* shares) in one module is the single source of truth for that decision, so the
|
|
11
|
+
* two paths can never drift apart.
|
|
12
|
+
*/
|
|
13
|
+
function isObject(value) {
|
|
14
|
+
return typeof value === "object" && value !== null;
|
|
15
|
+
}
|
|
16
|
+
function getErrorMessage(error) {
|
|
17
|
+
if (error instanceof Error) return error.message;
|
|
18
|
+
if (isObject(error) && typeof error.message === "string") return error.message;
|
|
19
|
+
return String(error);
|
|
20
|
+
}
|
|
21
|
+
function getErrorDiagnostic(error) {
|
|
22
|
+
const seen = /* @__PURE__ */ new Set();
|
|
23
|
+
const messages = [];
|
|
24
|
+
const stack = [error];
|
|
25
|
+
while (stack.length > 0) {
|
|
26
|
+
const current = stack.pop();
|
|
27
|
+
if (current === void 0 || seen.has(current)) continue;
|
|
28
|
+
seen.add(current);
|
|
29
|
+
const message = getErrorMessage(current);
|
|
30
|
+
if (message && message !== "[object Object]" && !messages.includes(message)) messages.push(message);
|
|
31
|
+
const code = getErrorCode(current);
|
|
32
|
+
if (code && !messages.includes(code)) messages.push(code);
|
|
33
|
+
stack.push(...getErrorChildren(current));
|
|
34
|
+
}
|
|
35
|
+
return messages.length > 0 ? messages.join(": ") : getErrorMessage(error);
|
|
36
|
+
}
|
|
37
|
+
function getErrorCode(error) {
|
|
38
|
+
if (!isObject(error)) return void 0;
|
|
39
|
+
const code = error.code ?? error.errno;
|
|
40
|
+
return typeof code === "string" ? code : void 0;
|
|
41
|
+
}
|
|
42
|
+
function getErrorStatus(error) {
|
|
43
|
+
if (!isObject(error)) return void 0;
|
|
44
|
+
const direct = error.status ?? error.statusCode;
|
|
45
|
+
if (typeof direct === "number") return direct;
|
|
46
|
+
if (isObject(error.response) && typeof error.response.status === "number") return error.response.status;
|
|
47
|
+
}
|
|
48
|
+
function getErrorChildren(error) {
|
|
49
|
+
if (!isObject(error)) return [];
|
|
50
|
+
const children = [];
|
|
51
|
+
if ("cause" in error) children.push(error.cause);
|
|
52
|
+
if (error instanceof AggregateError) children.push(...error.errors);
|
|
53
|
+
return children;
|
|
54
|
+
}
|
|
55
|
+
const CONNECTIVITY_ERROR_CODES = new Set([
|
|
56
|
+
"ECONNREFUSED",
|
|
57
|
+
"ECONNRESET",
|
|
58
|
+
"ENOTFOUND",
|
|
59
|
+
"ETIMEDOUT",
|
|
60
|
+
"EAI_AGAIN",
|
|
61
|
+
"EAI_NODATA",
|
|
62
|
+
"EAI_NONAME",
|
|
63
|
+
"EHOSTUNREACH",
|
|
64
|
+
"ENETUNREACH",
|
|
65
|
+
"CERT_HAS_EXPIRED",
|
|
66
|
+
"DEPTH_ZERO_SELF_SIGNED_CERT",
|
|
67
|
+
"ERR_TLS_CERT_ALTNAME_INVALID",
|
|
68
|
+
"SELF_SIGNED_CERT_IN_CHAIN",
|
|
69
|
+
"UNABLE_TO_VERIFY_LEAF_SIGNATURE"
|
|
70
|
+
]);
|
|
71
|
+
function isConnectivityMessage(message) {
|
|
72
|
+
return /\bconnection (?:refused|reset|timed out)\b/i.test(message) || /\bsocket hang up\b/i.test(message) || /\bnetwork error\b/i.test(message) || /\bcan'?t connect to\b/i.test(message) || /\bcertificate has expired\b/i.test(message) || /\bunable to verify the first certificate\b/i.test(message) || /\bupstream connect error or disconnect\/reset before headers\b/i.test(message);
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* True when a thrown failure is a transport/connectivity problem (DNS, refused
|
|
76
|
+
* connection, reset, TLS, 502/503/504, undici `UND_ERR_*`) rather than a
|
|
77
|
+
* deterministic error the warehouse returned. Walks `cause`/`AggregateError`
|
|
78
|
+
* chains so a wrapped connectivity error is still recognized. Callers degrade
|
|
79
|
+
* (and retry) on `true`; everything else is surfaced as a build failure.
|
|
80
|
+
*/
|
|
81
|
+
function isConnectivityError(error) {
|
|
82
|
+
const seen = /* @__PURE__ */ new Set();
|
|
83
|
+
const stack = [error];
|
|
84
|
+
while (stack.length > 0) {
|
|
85
|
+
const current = stack.pop();
|
|
86
|
+
if (current === void 0 || seen.has(current)) continue;
|
|
87
|
+
seen.add(current);
|
|
88
|
+
const code = getErrorCode(current);
|
|
89
|
+
if (code && (CONNECTIVITY_ERROR_CODES.has(code) || code.startsWith("UND_ERR_"))) return true;
|
|
90
|
+
const status = getErrorStatus(current);
|
|
91
|
+
if (status === 502 || status === 503 || status === 504) return true;
|
|
92
|
+
if (isConnectivityMessage(getErrorMessage(current))) return true;
|
|
93
|
+
stack.push(...getErrorChildren(current));
|
|
94
|
+
}
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
//#endregion
|
|
99
|
+
export { getErrorDiagnostic, getErrorMessage, isConnectivityError };
|
|
100
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","names":[],"sources":["../../src/type-generator/errors.ts"],"sourcesContent":["/**\n * Shared error-introspection utilities for the type generator.\n *\n * Both describe paths — the query registry and the metric-view registry — must\n * classify a thrown failure the same way: a transient connectivity blip (which\n * self-converges and should never fail a build) versus a deterministic failure\n * (auth, bad warehouse id, truncated result, malformed request) that must be\n * surfaced. Keeping `isConnectivityError` (and the message/diagnostic helpers it\n * shares) in one module is the single source of truth for that decision, so the\n * two paths can never drift apart.\n */\n\nfunction isObject(value: unknown): value is Record<string, unknown> {\n return typeof value === \"object\" && value !== null;\n}\n\nexport function getErrorMessage(error: unknown): string {\n if (error instanceof Error) return error.message;\n if (isObject(error) && typeof error.message === \"string\") {\n return error.message;\n }\n return String(error);\n}\n\nexport function getErrorDiagnostic(error: unknown): string {\n const seen = new Set<unknown>();\n const messages: string[] = [];\n const stack = [error];\n\n while (stack.length > 0) {\n const current = stack.pop();\n if (current === undefined || seen.has(current)) continue;\n seen.add(current);\n\n const message = getErrorMessage(current);\n if (\n message &&\n message !== \"[object Object]\" &&\n !messages.includes(message)\n ) {\n messages.push(message);\n }\n\n const code = getErrorCode(current);\n if (code && !messages.includes(code)) messages.push(code);\n\n stack.push(...getErrorChildren(current));\n }\n\n return messages.length > 0 ? messages.join(\": \") : getErrorMessage(error);\n}\n\nfunction getErrorCode(error: unknown): string | undefined {\n if (!isObject(error)) return undefined;\n const code = error.code ?? error.errno;\n return typeof code === \"string\" ? code : undefined;\n}\n\nfunction getErrorStatus(error: unknown): number | undefined {\n if (!isObject(error)) return undefined;\n const direct = error.status ?? error.statusCode;\n if (typeof direct === \"number\") return direct;\n if (isObject(error.response) && typeof error.response.status === \"number\") {\n return error.response.status;\n }\n return undefined;\n}\n\nfunction getErrorChildren(error: unknown): unknown[] {\n if (!isObject(error)) return [];\n const children: unknown[] = [];\n if (\"cause\" in error) children.push(error.cause);\n if (error instanceof AggregateError) children.push(...error.errors);\n return children;\n}\n\nconst CONNECTIVITY_ERROR_CODES = new Set([\n \"ECONNREFUSED\",\n \"ECONNRESET\",\n \"ENOTFOUND\",\n \"ETIMEDOUT\",\n \"EAI_AGAIN\",\n \"EAI_NODATA\",\n \"EAI_NONAME\",\n \"EHOSTUNREACH\",\n \"ENETUNREACH\",\n \"CERT_HAS_EXPIRED\",\n \"DEPTH_ZERO_SELF_SIGNED_CERT\",\n \"ERR_TLS_CERT_ALTNAME_INVALID\",\n \"SELF_SIGNED_CERT_IN_CHAIN\",\n \"UNABLE_TO_VERIFY_LEAF_SIGNATURE\",\n]);\n\nfunction isConnectivityMessage(message: string): boolean {\n return (\n /\\bconnection (?:refused|reset|timed out)\\b/i.test(message) ||\n /\\bsocket hang up\\b/i.test(message) ||\n /\\bnetwork error\\b/i.test(message) ||\n /\\bcan'?t connect to\\b/i.test(message) ||\n /\\bcertificate has expired\\b/i.test(message) ||\n /\\bunable to verify the first certificate\\b/i.test(message) ||\n /\\bupstream connect error or disconnect\\/reset before headers\\b/i.test(\n message,\n )\n );\n}\n\n/**\n * True when a thrown failure is a transport/connectivity problem (DNS, refused\n * connection, reset, TLS, 502/503/504, undici `UND_ERR_*`) rather than a\n * deterministic error the warehouse returned. Walks `cause`/`AggregateError`\n * chains so a wrapped connectivity error is still recognized. Callers degrade\n * (and retry) on `true`; everything else is surfaced as a build failure.\n */\nexport function isConnectivityError(error: unknown): boolean {\n const seen = new Set<unknown>();\n const stack = [error];\n\n while (stack.length > 0) {\n const current = stack.pop();\n if (current === undefined || seen.has(current)) continue;\n seen.add(current);\n\n const code = getErrorCode(current);\n if (\n code &&\n (CONNECTIVITY_ERROR_CODES.has(code) || code.startsWith(\"UND_ERR_\"))\n ) {\n return true;\n }\n\n const status = getErrorStatus(current);\n if (status === 502 || status === 503 || status === 504) return true;\n\n if (isConnectivityMessage(getErrorMessage(current))) return true;\n\n stack.push(...getErrorChildren(current));\n }\n\n return false;\n}\n"],"mappings":";;;;;;;;;;;;AAYA,SAAS,SAAS,OAAkD;AAClE,QAAO,OAAO,UAAU,YAAY,UAAU;;AAGhD,SAAgB,gBAAgB,OAAwB;AACtD,KAAI,iBAAiB,MAAO,QAAO,MAAM;AACzC,KAAI,SAAS,MAAM,IAAI,OAAO,MAAM,YAAY,SAC9C,QAAO,MAAM;AAEf,QAAO,OAAO,MAAM;;AAGtB,SAAgB,mBAAmB,OAAwB;CACzD,MAAM,uBAAO,IAAI,KAAc;CAC/B,MAAM,WAAqB,EAAE;CAC7B,MAAM,QAAQ,CAAC,MAAM;AAErB,QAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,YAAY,UAAa,KAAK,IAAI,QAAQ,CAAE;AAChD,OAAK,IAAI,QAAQ;EAEjB,MAAM,UAAU,gBAAgB,QAAQ;AACxC,MACE,WACA,YAAY,qBACZ,CAAC,SAAS,SAAS,QAAQ,CAE3B,UAAS,KAAK,QAAQ;EAGxB,MAAM,OAAO,aAAa,QAAQ;AAClC,MAAI,QAAQ,CAAC,SAAS,SAAS,KAAK,CAAE,UAAS,KAAK,KAAK;AAEzD,QAAM,KAAK,GAAG,iBAAiB,QAAQ,CAAC;;AAG1C,QAAO,SAAS,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,gBAAgB,MAAM;;AAG3E,SAAS,aAAa,OAAoC;AACxD,KAAI,CAAC,SAAS,MAAM,CAAE,QAAO;CAC7B,MAAM,OAAO,MAAM,QAAQ,MAAM;AACjC,QAAO,OAAO,SAAS,WAAW,OAAO;;AAG3C,SAAS,eAAe,OAAoC;AAC1D,KAAI,CAAC,SAAS,MAAM,CAAE,QAAO;CAC7B,MAAM,SAAS,MAAM,UAAU,MAAM;AACrC,KAAI,OAAO,WAAW,SAAU,QAAO;AACvC,KAAI,SAAS,MAAM,SAAS,IAAI,OAAO,MAAM,SAAS,WAAW,SAC/D,QAAO,MAAM,SAAS;;AAK1B,SAAS,iBAAiB,OAA2B;AACnD,KAAI,CAAC,SAAS,MAAM,CAAE,QAAO,EAAE;CAC/B,MAAM,WAAsB,EAAE;AAC9B,KAAI,WAAW,MAAO,UAAS,KAAK,MAAM,MAAM;AAChD,KAAI,iBAAiB,eAAgB,UAAS,KAAK,GAAG,MAAM,OAAO;AACnE,QAAO;;AAGT,MAAM,2BAA2B,IAAI,IAAI;CACvC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD,CAAC;AAEF,SAAS,sBAAsB,SAA0B;AACvD,QACE,8CAA8C,KAAK,QAAQ,IAC3D,sBAAsB,KAAK,QAAQ,IACnC,qBAAqB,KAAK,QAAQ,IAClC,yBAAyB,KAAK,QAAQ,IACtC,+BAA+B,KAAK,QAAQ,IAC5C,8CAA8C,KAAK,QAAQ,IAC3D,kEAAkE,KAChE,QACD;;;;;;;;;AAWL,SAAgB,oBAAoB,OAAyB;CAC3D,MAAM,uBAAO,IAAI,KAAc;CAC/B,MAAM,QAAQ,CAAC,MAAM;AAErB,QAAO,MAAM,SAAS,GAAG;EACvB,MAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,YAAY,UAAa,KAAK,IAAI,QAAQ,CAAE;AAChD,OAAK,IAAI,QAAQ;EAEjB,MAAM,OAAO,aAAa,QAAQ;AAClC,MACE,SACC,yBAAyB,IAAI,KAAK,IAAI,KAAK,WAAW,WAAW,EAElE,QAAO;EAGT,MAAM,SAAS,eAAe,QAAQ;AACtC,MAAI,WAAW,OAAO,WAAW,OAAO,WAAW,IAAK,QAAO;AAE/D,MAAI,sBAAsB,gBAAgB,QAAQ,CAAC,CAAE,QAAO;AAE5D,QAAM,KAAK,GAAG,iBAAiB,QAAQ,CAAC;;AAG1C,QAAO"}
|
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
import { createLogger } from "../logging/logger.js";
|
|
2
|
+
import { isRevivableMetricCacheEntry, loadCache, metricCacheHash, saveCache } from "./cache.js";
|
|
3
|
+
import { getErrorDiagnostic, isConnectivityError } from "./errors.js";
|
|
2
4
|
import { migrateProjectConfig, removeOldGeneratedTypes, resolveProjectRoot } from "./migration.js";
|
|
5
|
+
import { readMetricConfig, resolveMetricConfig } from "./mv-registry/config.js";
|
|
6
|
+
import { createWorkspaceDescribeFetcher } from "./mv-registry/describe.js";
|
|
7
|
+
import { generateMetricsMetadataJson } from "./mv-registry/metadata.js";
|
|
8
|
+
import { generateMetricTypeDeclarations } from "./mv-registry/render-types.js";
|
|
9
|
+
import { emptyMetricSchema, syncMetrics } from "./mv-registry/sync.js";
|
|
10
|
+
import { decidePreflight } from "./preflight.js";
|
|
11
|
+
import { getWarehouseState, startWarehouse, waitUntilRunning } from "./warehouse-status.js";
|
|
3
12
|
import { generateQueriesFromDescribe } from "./query-registry.js";
|
|
4
13
|
import { generateServingTypes as generateServingTypes$1 } from "./serving/generator.js";
|
|
14
|
+
import { WorkspaceClient } from "@databricks/sdk-experimental";
|
|
5
15
|
import pc from "picocolors";
|
|
6
16
|
import fs from "node:fs/promises";
|
|
7
17
|
import path from "node:path";
|
|
@@ -10,6 +20,12 @@ import dotenv from "dotenv";
|
|
|
10
20
|
//#region src/type-generator/index.ts
|
|
11
21
|
dotenv.config();
|
|
12
22
|
const logger = createLogger("type-generator");
|
|
23
|
+
/**
|
|
24
|
+
* Upper bound (~5 min) on how long the metric path's `blocking`-mode preflight
|
|
25
|
+
* waits for a warehouse to reach RUNNING. Mirrors the query path's (unexported)
|
|
26
|
+
* `PREFLIGHT_WAIT_MAX_MS` in query-registry.ts.
|
|
27
|
+
*/
|
|
28
|
+
const MV_PREFLIGHT_WAIT_MAX_MS = 3e5;
|
|
13
29
|
function plural(count, singular, pluralForm = `${singular}s`) {
|
|
14
30
|
return count === 1 ? singular : pluralForm;
|
|
15
31
|
}
|
|
@@ -131,14 +147,59 @@ declare module "@databricks/appkit-ui/react" {
|
|
|
131
147
|
`;
|
|
132
148
|
}
|
|
133
149
|
/**
|
|
150
|
+
* Status-only probe for the metric-view gate in {@link generateFromEntryPoint}:
|
|
151
|
+
* what state is the warehouse in right now?
|
|
152
|
+
*
|
|
153
|
+
* Uses {@link getWarehouseState} (`warehouses.get`) — a read-only GET that can
|
|
154
|
+
* never start the warehouse — unlike the metric DESCRIBE statements it guards,
|
|
155
|
+
* whose execution auto-starts a stopped warehouse and waits on it.
|
|
156
|
+
*
|
|
157
|
+
* Returns the observed state so the gate can distinguish a transient
|
|
158
|
+
* not-running state (STOPPED/STARTING/... → degraded entries that retry) from a
|
|
159
|
+
* terminal one (DELETED/DELETING → degraded entries pinned sticky). Takes the
|
|
160
|
+
* lazy client *getter* (not a client) so the probe also absorbs client
|
|
161
|
+
* construction failure. A connectivity blip returns `undefined`, which the gate
|
|
162
|
+
* reads as transient not-running; a deterministic failure (auth, bad id) is
|
|
163
|
+
* re-thrown so the gate can classify it fatal rather than silently degrading.
|
|
164
|
+
*/
|
|
165
|
+
async function probeWarehouseState(getClient, warehouseId) {
|
|
166
|
+
try {
|
|
167
|
+
return await getWarehouseState(getClient(), warehouseId);
|
|
168
|
+
} catch (err) {
|
|
169
|
+
if (isConnectivityError(err)) return void 0;
|
|
170
|
+
throw err;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
134
174
|
* Entry point for generating type declarations from all imported files
|
|
135
175
|
* @param options - the options for the generation
|
|
136
176
|
* @param options.entryPoint - the entry point file
|
|
137
177
|
* @param options.outFile - the output file
|
|
138
|
-
* @param options.
|
|
178
|
+
* @param options.noCache - skip the typegen cache entirely: every query is
|
|
179
|
+
* re-described, and the metric path ignores its cached schemas (every
|
|
180
|
+
* configured key becomes describe-needed) and overwrites the cache's
|
|
181
|
+
* `metrics` section with this pass's results.
|
|
182
|
+
* @param options.mode - preflight policy (see {@link PreflightMode}), default
|
|
183
|
+
* `"non-blocking"`. For queries, `"non-blocking"` never touches the
|
|
184
|
+
* warehouse. For metric views it makes one status-only probe and DESCRIBEs
|
|
185
|
+
* only when the warehouse is already RUNNING, otherwise emits permissive
|
|
186
|
+
* degraded types immediately. `"blocking"` waits for / starts the warehouse
|
|
187
|
+
* first, failing the build only for a deleted/deleting one.
|
|
188
|
+
* @param options.mvOutFile - optional output file for the MetricRegistry
|
|
189
|
+
* augmentation. Defaults to a sibling `metric.d.ts` file under the same
|
|
190
|
+
* directory as `outFile`. Skipped entirely if `metric-views.json` is absent.
|
|
191
|
+
* @param options.mvMetadataOutFile - optional output file for the
|
|
192
|
+
* build-time semantic metadata JSON bundle (`metrics.metadata.json`).
|
|
193
|
+
* Defaults to a sibling of `mvOutFile`. Skipped entirely if
|
|
194
|
+
* `metric-views.json` is absent.
|
|
195
|
+
* @param options.metricFetcher - optional DescribeFetcher used by
|
|
196
|
+
* {@link syncMetrics} (tests inject a mock; production lazily builds a
|
|
197
|
+
* default WorkspaceClient-backed one). An injected fetcher always runs: it
|
|
198
|
+
* hits no warehouse, so it bypasses both the non-blocking gate and the
|
|
199
|
+
* blocking preflight.
|
|
139
200
|
*/
|
|
140
201
|
async function generateFromEntryPoint(options) {
|
|
141
|
-
const { outFile, queryFolder, warehouseId, noCache, mode = "non-blocking" } = options;
|
|
202
|
+
const { outFile, queryFolder, warehouseId, noCache, mode = "non-blocking", mvOutFile, mvMetadataOutFile, metricFetcher } = options;
|
|
142
203
|
const projectRoot = resolveProjectRoot(outFile);
|
|
143
204
|
logger.debug("Starting type generation...");
|
|
144
205
|
let queryRegistry = [];
|
|
@@ -156,6 +217,121 @@ async function generateFromEntryPoint(options) {
|
|
|
156
217
|
const typeDeclarations = generateTypeDeclarations(queryRegistry);
|
|
157
218
|
await fs.mkdir(path.dirname(outFile), { recursive: true });
|
|
158
219
|
await fs.writeFile(outFile, typeDeclarations, "utf-8");
|
|
220
|
+
if (queryFolder) {
|
|
221
|
+
const mvConfig = await readMetricConfig(queryFolder);
|
|
222
|
+
if (mvConfig) {
|
|
223
|
+
const resolution = resolveMetricConfig(mvConfig);
|
|
224
|
+
const cache = await loadCache();
|
|
225
|
+
const mvCacheSection = Object.create(null);
|
|
226
|
+
if (!noCache && cache.metrics) for (const key of Object.keys(cache.metrics)) mvCacheSection[key] = cache.metrics[key];
|
|
227
|
+
const hitSchemas = /* @__PURE__ */ new Map();
|
|
228
|
+
const describeNeeded = [];
|
|
229
|
+
const stickyDegradedHits = [];
|
|
230
|
+
for (const entry of resolution.entries) {
|
|
231
|
+
const prior = mvCacheSection[entry.key];
|
|
232
|
+
if (prior !== void 0 && isRevivableMetricCacheEntry(prior) && prior.hash === metricCacheHash(entry.source, entry.lane) && !prior.retry) {
|
|
233
|
+
hitSchemas.set(entry.key, prior.schema);
|
|
234
|
+
if (prior.schema.degraded === true) stickyDegradedHits.push(entry.key);
|
|
235
|
+
} else describeNeeded.push(entry);
|
|
236
|
+
}
|
|
237
|
+
if (stickyDegradedHits.length > 0) logger.warn("cached failure for %s — fix the entry in metric-views.json or run with --no-cache to retry.", stickyDegradedHits.join(", "));
|
|
238
|
+
let mvClient;
|
|
239
|
+
const getMvClient = () => {
|
|
240
|
+
mvClient ??= new WorkspaceClient({});
|
|
241
|
+
return mvClient;
|
|
242
|
+
};
|
|
243
|
+
let preflightFatalMessage;
|
|
244
|
+
if (mode === "blocking" && metricFetcher === void 0 && describeNeeded.length > 0) try {
|
|
245
|
+
const state = await getWarehouseState(getMvClient(), warehouseId);
|
|
246
|
+
const decision = decidePreflight(state, mode);
|
|
247
|
+
if (decision === "fatal") preflightFatalMessage = `warehouse ${warehouseId} is ${state}`;
|
|
248
|
+
else if (decision === "startWaitProceed") {
|
|
249
|
+
await startWarehouse(getMvClient(), warehouseId);
|
|
250
|
+
const settled = await waitUntilRunning(getMvClient(), warehouseId, {
|
|
251
|
+
maxMs: MV_PREFLIGHT_WAIT_MAX_MS,
|
|
252
|
+
treatStoppedAsTransient: true
|
|
253
|
+
});
|
|
254
|
+
if (settled !== "RUNNING") preflightFatalMessage = `warehouse ${warehouseId} is ${settled}`;
|
|
255
|
+
} else if (decision === "waitThenProceed") {
|
|
256
|
+
const settled = await waitUntilRunning(getMvClient(), warehouseId, { maxMs: MV_PREFLIGHT_WAIT_MAX_MS });
|
|
257
|
+
if (settled === "DELETED" || settled === "DELETING") preflightFatalMessage = `warehouse ${warehouseId} is ${settled}`;
|
|
258
|
+
}
|
|
259
|
+
} catch (err) {
|
|
260
|
+
if (!isConnectivityError(err)) preflightFatalMessage = `warehouse ${warehouseId}: ${getErrorDiagnostic(err)}`;
|
|
261
|
+
}
|
|
262
|
+
let gateState;
|
|
263
|
+
let describeNow = metricFetcher !== void 0 || mode !== "non-blocking" || describeNeeded.length === 0;
|
|
264
|
+
if (!describeNow) {
|
|
265
|
+
try {
|
|
266
|
+
gateState = await probeWarehouseState(getMvClient, warehouseId);
|
|
267
|
+
} catch (err) {
|
|
268
|
+
preflightFatalMessage = `warehouse ${warehouseId}: ${getErrorDiagnostic(err)}`;
|
|
269
|
+
}
|
|
270
|
+
describeNow = gateState === "RUNNING";
|
|
271
|
+
}
|
|
272
|
+
let described;
|
|
273
|
+
let failures = [];
|
|
274
|
+
let terminalSkip = false;
|
|
275
|
+
if (preflightFatalMessage !== void 0) {
|
|
276
|
+
described = describeNeeded.map(emptyMetricSchema);
|
|
277
|
+
terminalSkip = true;
|
|
278
|
+
for (const entry of describeNeeded) fatalErrors.push({
|
|
279
|
+
name: entry.key,
|
|
280
|
+
message: preflightFatalMessage
|
|
281
|
+
});
|
|
282
|
+
} else if (describeNeeded.length === 0) described = [];
|
|
283
|
+
else if (describeNow) {
|
|
284
|
+
const fetcher = metricFetcher ?? createWorkspaceDescribeFetcher(getMvClient(), warehouseId);
|
|
285
|
+
({schemas: described, failures} = await syncMetrics({ entries: describeNeeded }, fetcher));
|
|
286
|
+
if (failures.length > 0) for (const f of failures) logger.warn("metric sync failed for %s (%s): %s", f.key, f.source, f.reason);
|
|
287
|
+
const failedKeys = new Set(failures.map((f) => f.key));
|
|
288
|
+
const degradedKeys = described.filter((s) => s.degraded && !failedKeys.has(s.key)).map((s) => s.key);
|
|
289
|
+
if (degradedKeys.length > 0) logger.info("Warehouse %s did not return schemas for %d metric view(s) (%s) — wrote degraded metric types (permissive); they will refresh once the warehouse is available.", warehouseId, degradedKeys.length, degradedKeys.join(", "));
|
|
290
|
+
} else {
|
|
291
|
+
described = describeNeeded.map(emptyMetricSchema);
|
|
292
|
+
terminalSkip = gateState === "DELETED" || gateState === "DELETING";
|
|
293
|
+
logger.info("Warehouse %s is not running — wrote degraded metric types (permissive) for %d metric view(s) (%s); they will refresh once the warehouse is available.", warehouseId, describeNeeded.length, describeNeeded.map((e) => e.key).join(", "));
|
|
294
|
+
}
|
|
295
|
+
const failureByKey = /* @__PURE__ */ new Map();
|
|
296
|
+
for (const failure of failures) failureByKey.set(failure.key, failure);
|
|
297
|
+
for (let i = 0; i < describeNeeded.length; i++) {
|
|
298
|
+
const entry = describeNeeded[i];
|
|
299
|
+
const failure = failureByKey.get(entry.key);
|
|
300
|
+
mvCacheSection[entry.key] = {
|
|
301
|
+
hash: metricCacheHash(entry.source, entry.lane),
|
|
302
|
+
schema: described[i],
|
|
303
|
+
retry: described[i].degraded === true && !terminalSkip && (failure === void 0 || failure.transient === true)
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
const configuredKeys = new Set(resolution.entries.map((e) => e.key));
|
|
307
|
+
let prunedCount = 0;
|
|
308
|
+
for (const key of Object.keys(mvCacheSection)) if (!configuredKeys.has(key)) {
|
|
309
|
+
delete mvCacheSection[key];
|
|
310
|
+
prunedCount++;
|
|
311
|
+
}
|
|
312
|
+
if (describeNeeded.length > 0 || noCache || prunedCount > 0) {
|
|
313
|
+
cache.metrics = mvCacheSection;
|
|
314
|
+
await saveCache(cache);
|
|
315
|
+
}
|
|
316
|
+
const describedByKey = /* @__PURE__ */ new Map();
|
|
317
|
+
for (const schema of described) describedByKey.set(schema.key, schema);
|
|
318
|
+
const mvSchemas = resolution.entries.map((entry) => {
|
|
319
|
+
const schema = hitSchemas.get(entry.key) ?? describedByKey.get(entry.key);
|
|
320
|
+
if (schema !== void 0) return schema;
|
|
321
|
+
logger.warn("no schema resolved for metric key %s — emitting degraded types (should not happen)", entry.key);
|
|
322
|
+
return emptyMetricSchema(entry);
|
|
323
|
+
});
|
|
324
|
+
const mvFile = mvOutFile ?? path.join(path.dirname(outFile), METRIC_TYPES_FILE);
|
|
325
|
+
const mvDeclarations = generateMetricTypeDeclarations(mvSchemas);
|
|
326
|
+
await fs.mkdir(path.dirname(mvFile), { recursive: true });
|
|
327
|
+
await fs.writeFile(mvFile, mvDeclarations, "utf-8");
|
|
328
|
+
const mvMetadataFile = mvMetadataOutFile ?? path.join(path.dirname(mvFile), METRIC_METADATA_FILE);
|
|
329
|
+
const metadataJson = generateMetricsMetadataJson(mvSchemas);
|
|
330
|
+
await fs.mkdir(path.dirname(mvMetadataFile), { recursive: true });
|
|
331
|
+
await fs.writeFile(mvMetadataFile, metadataJson, "utf-8");
|
|
332
|
+
logger.debug("Wrote MetricRegistry augmentation + metadata bundle for %d metric(s)%s", mvSchemas.length, failures.length > 0 ? ` (${failures.length} failure(s))` : "");
|
|
333
|
+
}
|
|
334
|
+
}
|
|
159
335
|
await removeOldGeneratedTypes(projectRoot, "appKitTypes.d.ts");
|
|
160
336
|
await migrateProjectConfig(projectRoot);
|
|
161
337
|
if (syntaxErrors.length > 0) throw new TypegenSyntaxError(syntaxErrors, warehouseId, fatalErrors);
|
|
@@ -169,6 +345,18 @@ const TYPES_DIR = "appkit-types";
|
|
|
169
345
|
const ANALYTICS_TYPES_FILE = "analytics.d.ts";
|
|
170
346
|
/** Default filename for serving endpoint type declarations. */
|
|
171
347
|
const SERVING_TYPES_FILE = "serving.d.ts";
|
|
348
|
+
/** Default filename for metric-view registry type declarations. */
|
|
349
|
+
const METRIC_TYPES_FILE = "metric.d.ts";
|
|
350
|
+
/**
|
|
351
|
+
* Default filename for the build-time semantic-metadata JSON bundle, sibling of
|
|
352
|
+
* {@link METRIC_TYPES_FILE}. Shape is `Record<metricKey, { measures,
|
|
353
|
+
* dimensions }>` (UC FQN and execution lane are server-side concerns, kept out
|
|
354
|
+
* of this client-shipped artifact). The consuming app imports it at build time
|
|
355
|
+
* and registers it via `@databricks/appkit-ui/format`'s
|
|
356
|
+
* `registerMetricsMetadata()`, so the React hook returns per-metric `metadata`
|
|
357
|
+
* without a second network round-trip.
|
|
358
|
+
*/
|
|
359
|
+
const METRIC_METADATA_FILE = "metrics.metadata.json";
|
|
172
360
|
|
|
173
361
|
//#endregion
|
|
174
362
|
export { ANALYTICS_TYPES_FILE, SERVING_TYPES_FILE, TYPES_DIR, TypegenFatalError, TypegenSyntaxError, generateFromEntryPoint, generateServingTypes };
|