@google-cloud/kcmd 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -20
- package/build/ts/kcmd/cli.js +5842 -5168
- package/build/ts/kcmd/gcp/biglake.d.ts +2 -0
- package/build/ts/kcmd/gcp/biglake.js +13 -4
- package/build/ts/kcmd/gcp/spanner.d.ts +10 -0
- package/build/ts/kcmd/gcp/spanner.js +6 -3
- package/build/ts/kcmd/layouts/okf.js +49 -7
- package/build/ts/kcmd/layouts/semantic-model.d.ts +13 -0
- package/build/ts/kcmd/layouts/semantic-model.js +129 -50
- package/build/ts/kcmd/okf/concept.d.ts +1 -1
- package/build/ts/kcmd/okf/concept.js +6 -0
- package/build/ts/kcmd/okf/concept_to_entry.d.ts +2 -0
- package/build/ts/kcmd/okf/concept_to_entry.js +29 -2
- package/build/ts/kcmd/okf/init_okf_workspace.js +8 -10
- package/build/ts/kcmd/okf/okf_metadata.js +17 -0
- package/build/ts/kcmd/semantic/bigquery.js +139 -195
- package/build/ts/kcmd/semantic/binding.js +14 -19
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.d.ts +10 -35
- package/build/ts/kcmd/semantic/deploy_knowledge_catalog.js +92 -121
- package/build/ts/kcmd/semantic/graph_ddl.d.ts +10 -0
- package/build/ts/kcmd/semantic/graph_ddl.js +152 -59
- package/build/ts/kcmd/semantic/import_model.d.ts +26 -4
- package/build/ts/kcmd/semantic/import_model.js +526 -103
- package/build/ts/kcmd/semantic/kc_converter.d.ts +7 -1
- package/build/ts/kcmd/semantic/kc_converter.js +20 -5
- package/build/ts/kcmd/semantic/kc_custom_types.d.ts +5 -12
- package/build/ts/kcmd/semantic/kc_custom_types.js +16 -93
- package/build/ts/kcmd/semantic/kc_ids.d.ts +4 -5
- package/build/ts/kcmd/semantic/kc_ids.js +4 -7
- package/build/ts/kcmd/semantic/knowledge_catalog.d.ts +0 -3
- package/build/ts/kcmd/semantic/knowledge_catalog.js +3 -86
- package/build/ts/kcmd/semantic/loader.d.ts +7 -11
- package/build/ts/kcmd/semantic/loader.js +284 -148
- package/build/ts/kcmd/semantic/osi_converter.js +1 -2
- package/build/ts/kcmd/semantic/parameters.d.ts +1 -0
- package/build/ts/kcmd/semantic/parameters.js +22 -3
- package/build/ts/kcmd/semantic/pull_kc.d.ts +1 -5
- package/build/ts/kcmd/semantic/pull_kc.js +38 -112
- package/build/ts/kcmd/semantic/resolve_inheritance.d.ts +7 -2
- package/build/ts/kcmd/semantic/resolve_inheritance.js +37 -21
- package/build/ts/kcmd/semantic/resolve_profiles.d.ts +51 -10
- package/build/ts/kcmd/semantic/resolve_profiles.js +213 -32
- package/build/ts/kcmd/semantic/runtime/runtime.d.ts +0 -6
- package/build/ts/kcmd/semantic/runtime/runtime.js +4 -21
- package/build/ts/kcmd/semantic/source_dialect.d.ts +9 -2
- package/build/ts/kcmd/semantic/source_dialect.js +83 -33
- package/build/ts/kcmd/semantic/source_entries.d.ts +1 -1
- package/build/ts/kcmd/semantic/source_entries.js +95 -36
- package/build/ts/kcmd/semantic/sql_expr_utils.d.ts +13 -7
- package/build/ts/kcmd/semantic/sql_expr_utils.js +240 -67
- package/build/ts/kcmd/semantic/sql_parser.d.ts +5 -2
- package/build/ts/kcmd/semantic/sql_parser.js +22 -0
- package/build/ts/kcmd/semantic/transpile.d.ts +12 -10
- package/build/ts/kcmd/semantic/transpile.js +79 -61
- package/build/ts/kcmd/semantic/validate.d.ts +37 -2
- package/build/ts/kcmd/semantic/validate.js +465 -98
- package/build/ts/kcmd/snapshot.d.ts +4 -0
- package/build/ts/kcmd/snapshot.js +7 -0
- package/build/ts/kcmd/sync.js +44 -7
- package/package.json +1 -1
|
@@ -11,6 +11,8 @@ export interface Catalog {
|
|
|
11
11
|
[key: string]: any;
|
|
12
12
|
}
|
|
13
13
|
export declare class BigLakeClient extends api.ApiClient {
|
|
14
|
+
private readonly hive;
|
|
14
15
|
constructor(ctx: context.ApiContext);
|
|
15
16
|
getCatalog(project: string, catalog: string): Promise<api.ApiResult<Catalog>>;
|
|
17
|
+
getHiveCatalog(project: string, catalog: string): Promise<api.ApiResult<Catalog>>;
|
|
16
18
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
// API client for BigLake, covering the
|
|
3
|
-
//
|
|
4
|
-
// Catalog keeps the entries of its tables.
|
|
2
|
+
// API client for BigLake, covering the calls the source check needs: reading
|
|
3
|
+
// an Iceberg REST catalog or a Hive catalog, whose region decides where
|
|
4
|
+
// Knowledge Catalog keeps the entries of its tables.
|
|
5
5
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
6
6
|
if (k2 === undefined) k2 = k;
|
|
7
7
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
@@ -39,12 +39,21 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
39
39
|
exports.BigLakeClient = void 0;
|
|
40
40
|
const api = __importStar(require("./api"));
|
|
41
41
|
class BigLakeClient extends api.ApiClient {
|
|
42
|
+
hive;
|
|
42
43
|
constructor(ctx) {
|
|
43
44
|
super('https://biglake.googleapis.com', 'iceberg/v1/restcatalog/extensions', ctx);
|
|
45
|
+
this.hive =
|
|
46
|
+
new api.ApiClient('https://biglake.googleapis.com', 'hive/v1', ctx);
|
|
44
47
|
}
|
|
45
|
-
// Fetches
|
|
48
|
+
// Fetches an Iceberg REST catalog. A catalog that does not exist answers
|
|
49
|
+
// 404, and a Hive catalog answers 400.
|
|
46
50
|
async getCatalog(project, catalog) {
|
|
47
51
|
return await this._get(`projects/${project}/catalogs/${catalog}`);
|
|
48
52
|
}
|
|
53
|
+
// Fetches a Hive catalog. Any other catalog answers 404, and an id that is
|
|
54
|
+
// not lower-case letters, digits and underscores answers 400.
|
|
55
|
+
async getHiveCatalog(project, catalog) {
|
|
56
|
+
return await this.hive._get(`projects/${project}/catalogs/${catalog}`);
|
|
57
|
+
}
|
|
49
58
|
}
|
|
50
59
|
exports.BigLakeClient = BigLakeClient;
|
|
@@ -35,7 +35,16 @@ export interface SpannerType {
|
|
|
35
35
|
}
|
|
36
36
|
export interface Instance {
|
|
37
37
|
name?: string;
|
|
38
|
+
edition?: string;
|
|
38
39
|
config?: string;
|
|
40
|
+
replicaComputeCapacity?: Array<{
|
|
41
|
+
replicaSelection?: {
|
|
42
|
+
location?: string;
|
|
43
|
+
};
|
|
44
|
+
processingUnits?: number;
|
|
45
|
+
nodeCount?: number;
|
|
46
|
+
}>;
|
|
47
|
+
resourceLocation?: string;
|
|
39
48
|
[key: string]: any;
|
|
40
49
|
}
|
|
41
50
|
export interface InstanceConfig {
|
|
@@ -52,6 +61,7 @@ export declare class SpannerClient extends api.ApiClient {
|
|
|
52
61
|
updateDatabaseDdl(project: string, instance: string, database: string, statements: string[]): Promise<api.ApiResult<Operation>>;
|
|
53
62
|
getDatabase(project: string, instance: string, database: string): Promise<api.ApiResult<{
|
|
54
63
|
name?: string;
|
|
64
|
+
databaseDialect?: string;
|
|
55
65
|
[key: string]: any;
|
|
56
66
|
}>>;
|
|
57
67
|
getInstance(project: string, instance: string): Promise<api.ApiResult<Instance>>;
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// database and deploy Spanner Graph DDL; the data plane (sessions and
|
|
5
5
|
// executeSql) to PLAN an action's DML against the live database before
|
|
6
6
|
// publishing it; and Instance Admin (getInstance and getInstanceConfig) to
|
|
7
|
-
// find the region where Knowledge Catalog keeps a table's entry
|
|
7
|
+
// find the region where Knowledge Catalog keeps a table's entry and to read an
|
|
8
|
+
// instance's edition.
|
|
8
9
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
10
|
if (k2 === undefined) k2 = k;
|
|
10
11
|
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
@@ -54,11 +55,13 @@ class SpannerClient extends api.ApiClient {
|
|
|
54
55
|
const name = `projects/${project}/instances/${instance}/databases/${database}/ddl`;
|
|
55
56
|
return await this._patch(name, { statements });
|
|
56
57
|
}
|
|
57
|
-
// Fetches a database's metadata
|
|
58
|
+
// Fetches a database's metadata, which includes its databaseDialect:
|
|
59
|
+
// GOOGLE_STANDARD_SQL or POSTGRESQL.
|
|
58
60
|
async getDatabase(project, instance, database) {
|
|
59
61
|
return await this._get(`projects/${project}/instances/${instance}/databases/${database}`);
|
|
60
62
|
}
|
|
61
|
-
// Fetches an instance's metadata, which names its instance config
|
|
63
|
+
// Fetches an instance's metadata, which names its instance config and its
|
|
64
|
+
// edition.
|
|
62
65
|
async getInstance(project, instance) {
|
|
63
66
|
return await this._get(`projects/${project}/instances/${instance}`);
|
|
64
67
|
}
|
|
@@ -51,14 +51,36 @@ function mapping(value) {
|
|
|
51
51
|
function text(value) {
|
|
52
52
|
return typeof value === 'string' && value !== '' ? value : undefined;
|
|
53
53
|
}
|
|
54
|
-
function conceptFiles(dir) {
|
|
54
|
+
function conceptFiles(dir, realRoot) {
|
|
55
|
+
const root = realRoot ?? fs.realpathSync(dir);
|
|
55
56
|
return fs
|
|
56
57
|
.readdirSync(dir, { withFileTypes: true })
|
|
57
58
|
.sort((a, b) => a.name.localeCompare(b.name))
|
|
58
59
|
.flatMap((e) => {
|
|
60
|
+
// OKF v0.3 §3.2(4): dot-prefixed files and directories are not part of the bundle.
|
|
61
|
+
if (e.name.startsWith('.'))
|
|
62
|
+
return [];
|
|
59
63
|
const p = path.join(dir, e.name);
|
|
64
|
+
if (e.isSymbolicLink()) {
|
|
65
|
+
// OKF v0.3 §3.2(5): do not follow symlinks that resolve outside the bundle root.
|
|
66
|
+
let realTarget;
|
|
67
|
+
let stat;
|
|
68
|
+
try {
|
|
69
|
+
realTarget = fs.realpathSync(p);
|
|
70
|
+
stat = fs.statSync(p);
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
if (realTarget !== root && !realTarget.startsWith(root + path.sep)) {
|
|
76
|
+
return [];
|
|
77
|
+
}
|
|
78
|
+
if (stat.isDirectory())
|
|
79
|
+
return conceptFiles(p, root);
|
|
80
|
+
return e.name.endsWith('.md') ? [p] : [];
|
|
81
|
+
}
|
|
60
82
|
if (e.isDirectory())
|
|
61
|
-
return conceptFiles(p);
|
|
83
|
+
return conceptFiles(p, root);
|
|
62
84
|
return e.name.endsWith('.md') ? [p] : [];
|
|
63
85
|
});
|
|
64
86
|
}
|
|
@@ -67,8 +89,9 @@ function toPosixId(root, file) {
|
|
|
67
89
|
}
|
|
68
90
|
/**
|
|
69
91
|
* Resolves the catalog entry ID a concept file maps to, or `undefined` when the
|
|
70
|
-
* file should be skipped (`<group>_entry`,
|
|
71
|
-
*
|
|
92
|
+
* file should be skipped (`<group>_entry`, reserved `log.md` at any level,
|
|
93
|
+
* untyped non-concept `.md` files per OKF v0.3 §3.2, or a generated `index.md`
|
|
94
|
+
* shadowed by an explicit `agents.md` / `<dir>.md`).
|
|
72
95
|
*/
|
|
73
96
|
function resolvedEntryName(id, frontmatter, bundlePath) {
|
|
74
97
|
const residue = mapping(frontmatter[concept_1.RESIDUE_KEY]);
|
|
@@ -77,11 +100,17 @@ function resolvedEntryName(id, frontmatter, bundlePath) {
|
|
|
77
100
|
text(extra['type'])?.endsWith('/entrygroup')) {
|
|
78
101
|
return undefined;
|
|
79
102
|
}
|
|
103
|
+
const hasConceptType = text(frontmatter['type']) !== undefined ||
|
|
104
|
+
text(residue['entryType']) !== undefined ||
|
|
105
|
+
text(extra['type']) !== undefined;
|
|
80
106
|
const hasExplicitMeta = frontmatter[concept_1.RESIDUE_KEY] !== undefined ||
|
|
81
107
|
frontmatter[concept_1.EXTRA_KEY] !== undefined ||
|
|
82
|
-
|
|
83
|
-
//
|
|
84
|
-
|
|
108
|
+
hasConceptType;
|
|
109
|
+
// OKF v0.3 §§3.1, 9: `log.md` at any level is a reserved history file, never
|
|
110
|
+
// a concept, unless it is a pulled remote entry carrying `x-kcmd` / `extra`.
|
|
111
|
+
if ((id === 'log' || id.endsWith('/log')) &&
|
|
112
|
+
frontmatter[concept_1.RESIDUE_KEY] === undefined &&
|
|
113
|
+
frontmatter[concept_1.EXTRA_KEY] === undefined) {
|
|
85
114
|
return undefined;
|
|
86
115
|
}
|
|
87
116
|
// When `agents.md` or `<dir>.md` already exists alongside a directory-listing
|
|
@@ -96,6 +125,13 @@ function resolvedEntryName(id, frontmatter, bundlePath) {
|
|
|
96
125
|
fs.existsSync(path.join(bundlePath, `${id.slice(0, -'/index'.length)}.md`))) {
|
|
97
126
|
return undefined;
|
|
98
127
|
}
|
|
128
|
+
// OKF v0.3 §3.2(2): a non-index `.md` file with no `type` in frontmatter
|
|
129
|
+
// (e.g. `README.md`, `AGENTS.md`, `CONTRIBUTING.md`, `SKILL.md`) is not a
|
|
130
|
+
// concept; consumers skip it rather than rejecting the bundle.
|
|
131
|
+
const isIndex = id === 'index' || id.endsWith('/index');
|
|
132
|
+
if (!isIndex && !hasConceptType) {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
99
135
|
const explicitId = text(residue['entryId']) ?? text(extra['entry_id']);
|
|
100
136
|
if (explicitId)
|
|
101
137
|
return explicitId;
|
|
@@ -292,10 +328,16 @@ class OkfLayout {
|
|
|
292
328
|
return (0, concept_to_entry_1.entryToConcept)(gcpEntry, name);
|
|
293
329
|
}
|
|
294
330
|
async saveEntry(name, entry) {
|
|
331
|
+
const prevFile = this._index.get(name);
|
|
295
332
|
const file = this.resolveConceptPath(name);
|
|
296
333
|
const concept = this.previewConcept(name, entry);
|
|
297
334
|
await fs.promises.mkdir(path.dirname(file), { recursive: true });
|
|
298
335
|
await fs.promises.writeFile(file, (0, concept_1.conceptToFileContent)(concept), 'utf8');
|
|
336
|
+
if (prevFile &&
|
|
337
|
+
path.resolve(prevFile) !== path.resolve(file) &&
|
|
338
|
+
fs.existsSync(prevFile)) {
|
|
339
|
+
await fs.promises.unlink(prevFile);
|
|
340
|
+
}
|
|
299
341
|
this._index.set(name, file);
|
|
300
342
|
}
|
|
301
343
|
async deleteEntry(name) {
|
|
@@ -4,6 +4,7 @@ export declare function profileFilesBeside(dir: string, model: string): {
|
|
|
4
4
|
name: string;
|
|
5
5
|
path: string;
|
|
6
6
|
}[];
|
|
7
|
+
export declare function profileFileNamesBeside(dir: string, model: string): string[];
|
|
7
8
|
export interface OverviewSidecar {
|
|
8
9
|
content?: string;
|
|
9
10
|
contentType?: string;
|
|
@@ -12,6 +13,12 @@ export interface OverviewSidecar {
|
|
|
12
13
|
title?: string;
|
|
13
14
|
}>;
|
|
14
15
|
}
|
|
16
|
+
export declare function vanillaSiblingError(file: string, modelFile: string): string;
|
|
17
|
+
export declare function legacyProfileFiles(dir: string, model: string): {
|
|
18
|
+
name: string;
|
|
19
|
+
path: string;
|
|
20
|
+
}[];
|
|
21
|
+
export declare function legacyProfileDirError(model: string): string;
|
|
15
22
|
export declare class SemanticModelLayout implements CatalogLayout {
|
|
16
23
|
private readonly _catalogPath;
|
|
17
24
|
private readonly _entryGroup?;
|
|
@@ -28,6 +35,7 @@ export declare class SemanticModelLayout implements CatalogLayout {
|
|
|
28
35
|
path: string;
|
|
29
36
|
profileName?: string;
|
|
30
37
|
}[];
|
|
38
|
+
profileFilePaths(model: string): string[];
|
|
31
39
|
overviewSidecars(): Map<string, OverviewSidecar>;
|
|
32
40
|
overviewSidecarPaths(): {
|
|
33
41
|
entryId: string;
|
|
@@ -59,3 +67,8 @@ export declare class SemanticModelLayout implements CatalogLayout {
|
|
|
59
67
|
export declare function fileNameOf(layout: SemanticModelLayout, model: string, localPath: string): string;
|
|
60
68
|
export declare function parseOverviewSidecar(text: string, file: string): OverviewSidecar;
|
|
61
69
|
export declare function formatOverviewSidecar({ content, ...rest }: OverviewSidecar): string;
|
|
70
|
+
export declare class ProfileFileError extends Error {
|
|
71
|
+
}
|
|
72
|
+
export declare function checkProfileFileName(modelFile: string, file: string, name: string, text: string, seen: Map<string, string>): void;
|
|
73
|
+
export declare function profileNameError(file: string, name: string): string | undefined;
|
|
74
|
+
export declare function profileParseError(file: string, message: string): string;
|
|
@@ -42,11 +42,18 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
42
42
|
};
|
|
43
43
|
})();
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
-
exports.SemanticModelLayout = void 0;
|
|
45
|
+
exports.ProfileFileError = exports.SemanticModelLayout = void 0;
|
|
46
46
|
exports.profileFilesBeside = profileFilesBeside;
|
|
47
|
+
exports.profileFileNamesBeside = profileFileNamesBeside;
|
|
48
|
+
exports.vanillaSiblingError = vanillaSiblingError;
|
|
49
|
+
exports.legacyProfileFiles = legacyProfileFiles;
|
|
50
|
+
exports.legacyProfileDirError = legacyProfileDirError;
|
|
47
51
|
exports.fileNameOf = fileNameOf;
|
|
48
52
|
exports.parseOverviewSidecar = parseOverviewSidecar;
|
|
49
53
|
exports.formatOverviewSidecar = formatOverviewSidecar;
|
|
54
|
+
exports.checkProfileFileName = checkProfileFileName;
|
|
55
|
+
exports.profileNameError = profileNameError;
|
|
56
|
+
exports.profileParseError = profileParseError;
|
|
50
57
|
const glob = __importStar(require("glob"));
|
|
51
58
|
const fs = __importStar(require("node:fs"));
|
|
52
59
|
const path = __importStar(require("node:path"));
|
|
@@ -82,6 +89,22 @@ function profileFilesBeside(dir, model) {
|
|
|
82
89
|
}
|
|
83
90
|
return out.sort((a, b) => a.name.localeCompare(b.name));
|
|
84
91
|
}
|
|
92
|
+
// Each `<model>.profile.*.yaml` in `dir`, whatever its profile name: the
|
|
93
|
+
// nameless file first, then those whose profile name is empty or has a dot, by
|
|
94
|
+
// file name, then the rest by profile name.
|
|
95
|
+
function profileFileNamesBeside(dir, model) {
|
|
96
|
+
if (!fs.existsSync(dir))
|
|
97
|
+
return [];
|
|
98
|
+
const named = profileFilesBeside(dir, model).map(p => path.basename(p.path));
|
|
99
|
+
const nameless = `${model}.profile.yaml`;
|
|
100
|
+
const others = fs.readdirSync(dir).filter(f => f.startsWith(`${model}.profile.`) && f.endsWith('.yaml') &&
|
|
101
|
+
f !== nameless && !named.includes(f));
|
|
102
|
+
return [
|
|
103
|
+
...(fs.existsSync(path.join(dir, nameless)) ? [nameless] : []),
|
|
104
|
+
...others.sort(),
|
|
105
|
+
...named,
|
|
106
|
+
];
|
|
107
|
+
}
|
|
85
108
|
// The suffix of an entry's overview sidecar.
|
|
86
109
|
const OVERVIEW_SIDECAR = '.overview.md';
|
|
87
110
|
// The profile name reserved for the inline bindings in the model file.
|
|
@@ -89,14 +112,36 @@ const DEFAULT_PROFILE_NAME = 'default';
|
|
|
89
112
|
// The flavor that keeps its profiles inline, in a GOOGLE block, and so has no
|
|
90
113
|
// sibling profile files.
|
|
91
114
|
const VANILLA_VERSION = '0.2.0.dev0';
|
|
92
|
-
//
|
|
93
|
-
// profiles in its GOOGLE block.
|
|
94
|
-
function vanillaSiblingError(file,
|
|
95
|
-
return `Profile file '${file}' sits beside '${
|
|
115
|
+
// Push's error for the profile file `file` beside the vanilla model file
|
|
116
|
+
// `modelFile`, which keeps its profiles in its GOOGLE block.
|
|
117
|
+
function vanillaSiblingError(file, modelFile) {
|
|
118
|
+
return `Profile file '${file}' sits beside '${modelFile}', which is a ` +
|
|
96
119
|
`${VANILLA_VERSION} document; sibling profile files are Google-flavor ` +
|
|
97
120
|
`only, and a ${VANILLA_VERSION} model keeps its profiles in its GOOGLE ` +
|
|
98
121
|
`block.`;
|
|
99
122
|
}
|
|
123
|
+
// The profile files in the older `<model>.profiles/` directory in `dir`, each
|
|
124
|
+
// named by its file basename.
|
|
125
|
+
function legacyProfileFiles(dir, model) {
|
|
126
|
+
const legacy = path.join(dir, `${model}.profiles`);
|
|
127
|
+
if (!fs.statSync(legacy, { throwIfNoEntry: false })?.isDirectory())
|
|
128
|
+
return [];
|
|
129
|
+
const out = [];
|
|
130
|
+
for (const entry of fs.readdirSync(legacy)) {
|
|
131
|
+
if (!entry.endsWith('.yaml'))
|
|
132
|
+
continue;
|
|
133
|
+
if (SIDECAR_SUFFIXES.some(s => entry.endsWith(s)))
|
|
134
|
+
continue;
|
|
135
|
+
out.push({ name: path.basename(entry, '.yaml'), path: path.join(legacy, entry) });
|
|
136
|
+
}
|
|
137
|
+
return out.sort((a, b) => a.name.localeCompare(b.name));
|
|
138
|
+
}
|
|
139
|
+
// Push's error for the older `<model>.profiles/` directory.
|
|
140
|
+
function legacyProfileDirError(model) {
|
|
141
|
+
return `Legacy profile directory '${model}.profiles/' is no longer ` +
|
|
142
|
+
`supported; rename profile files to '${model}.profile.<name>.yaml' ` +
|
|
143
|
+
`beside '${model}.yaml'.`;
|
|
144
|
+
}
|
|
100
145
|
class SemanticModelLayout {
|
|
101
146
|
_catalogPath;
|
|
102
147
|
_entryGroup;
|
|
@@ -179,6 +224,18 @@ class SemanticModelLayout {
|
|
|
179
224
|
...this._legacyProfilePaths(model).map(p => ({ path: p.path })),
|
|
180
225
|
];
|
|
181
226
|
}
|
|
227
|
+
// Every profile file of a model on disk, whatever its profile name: each
|
|
228
|
+
// `<model>.profile.*.yaml` in the entry group, as profileFileNamesBeside
|
|
229
|
+
// lists them, then each file in the older `<model>.profiles/` directory.
|
|
230
|
+
profileFilePaths(model) {
|
|
231
|
+
if (!this._entryGroup)
|
|
232
|
+
return [];
|
|
233
|
+
const dir = this._groupDir();
|
|
234
|
+
return [
|
|
235
|
+
...profileFileNamesBeside(dir, model).map(f => path.join(dir, f)),
|
|
236
|
+
...this._legacyProfilePaths(model).map(p => p.path),
|
|
237
|
+
];
|
|
238
|
+
}
|
|
182
239
|
// The overview sidecar of each entry, by entry id. Throws naming a sidecar
|
|
183
240
|
// parseOverviewSidecar rejects.
|
|
184
241
|
overviewSidecars() {
|
|
@@ -218,8 +275,7 @@ class SemanticModelLayout {
|
|
|
218
275
|
path.join(this._groupDir(), `${model}.profile.yaml`) :
|
|
219
276
|
undefined;
|
|
220
277
|
if (nameless && fs.existsSync(nameless)) {
|
|
221
|
-
throw new Error(
|
|
222
|
-
`rename it '${model}.profile.<name>.yaml'.`);
|
|
278
|
+
throw new Error(profileNameError(path.basename(nameless), ''));
|
|
223
279
|
}
|
|
224
280
|
const prefix = `${model}.profile.`;
|
|
225
281
|
const badName = this._entryGroup && fs.existsSync(this._groupDir()) ?
|
|
@@ -232,9 +288,7 @@ class SemanticModelLayout {
|
|
|
232
288
|
}) :
|
|
233
289
|
undefined;
|
|
234
290
|
if (badName) {
|
|
235
|
-
throw new Error(
|
|
236
|
-
`name is a letter followed by letters, digits, underscores and ` +
|
|
237
|
-
`hyphens.`);
|
|
291
|
+
throw new Error(profileNameError(badName, badName.slice(prefix.length, -'.yaml'.length)));
|
|
238
292
|
}
|
|
239
293
|
const siblings = this._siblingProfilePaths(model);
|
|
240
294
|
if (!siblings.length)
|
|
@@ -253,40 +307,17 @@ class SemanticModelLayout {
|
|
|
253
307
|
}
|
|
254
308
|
const version = header.version;
|
|
255
309
|
const docs = [];
|
|
256
|
-
|
|
257
|
-
const byLowerName = new Map();
|
|
310
|
+
const seen = new Map();
|
|
258
311
|
for (const { name, path: p } of siblings) {
|
|
259
312
|
const file = path.basename(p);
|
|
260
|
-
const twin = byLowerName.get(name.toLowerCase());
|
|
261
|
-
if (twin) {
|
|
262
|
-
throw new Error(`Profile files '${twin}' and '${file}' name the same profile; ` +
|
|
263
|
-
`profile names are unique ignoring case.`);
|
|
264
|
-
}
|
|
265
|
-
byLowerName.set(name.toLowerCase(), file);
|
|
266
313
|
if (!fs.statSync(p).isFile()) {
|
|
267
314
|
throw new Error(`Profile file '${file}' is not a file.`);
|
|
268
315
|
}
|
|
269
316
|
if (version === VANILLA_VERSION) {
|
|
270
|
-
throw new Error(vanillaSiblingError(file,
|
|
317
|
+
throw new Error(vanillaSiblingError(file, modelFile));
|
|
271
318
|
}
|
|
272
319
|
const text = fs.readFileSync(p, 'utf8');
|
|
273
|
-
|
|
274
|
-
const reserved = (n) => n?.toLowerCase() === DEFAULT_PROFILE_NAME;
|
|
275
|
-
if (reserved(name)) {
|
|
276
|
-
throw new Error(`Profile name '${DEFAULT_PROFILE_NAME}' is reserved for the ` +
|
|
277
|
-
`inline bindings in '${modelFile}'; remove '${file}'.`);
|
|
278
|
-
}
|
|
279
|
-
if (reserved(declared)) {
|
|
280
|
-
throw new Error(`Profile name '${DEFAULT_PROFILE_NAME}' is reserved for the ` +
|
|
281
|
-
`inline bindings in '${modelFile}'; change the name in '${file}' to '${name}'.`);
|
|
282
|
-
}
|
|
283
|
-
if (declared === undefined) {
|
|
284
|
-
throw new Error(`Profile file '${file}' declares no name; add 'name: ${name}'.`);
|
|
285
|
-
}
|
|
286
|
-
if (declared !== name) {
|
|
287
|
-
throw new Error(`Profile file '${file}' declares name '${declared}', which ` +
|
|
288
|
-
`does not match filename suffix '${name}'.`);
|
|
289
|
-
}
|
|
320
|
+
checkProfileFileName(modelFile, file, name, text, seen);
|
|
290
321
|
docs.push({ name, text, path: p });
|
|
291
322
|
}
|
|
292
323
|
return docs;
|
|
@@ -376,18 +407,7 @@ class SemanticModelLayout {
|
|
|
376
407
|
_legacyProfilePaths(model) {
|
|
377
408
|
if (!this._entryGroup || /[\\/]/.test(model))
|
|
378
409
|
return [];
|
|
379
|
-
|
|
380
|
-
if (!fs.existsSync(dir))
|
|
381
|
-
return [];
|
|
382
|
-
const out = [];
|
|
383
|
-
for (const entry of fs.readdirSync(dir)) {
|
|
384
|
-
if (!entry.endsWith('.yaml'))
|
|
385
|
-
continue;
|
|
386
|
-
if (SIDECAR_SUFFIXES.some(s => entry.endsWith(s)))
|
|
387
|
-
continue;
|
|
388
|
-
out.push({ name: path.basename(entry, '.yaml'), path: path.join(dir, entry) });
|
|
389
|
-
}
|
|
390
|
-
return out.sort((a, b) => a.name.localeCompare(b.name));
|
|
410
|
+
return legacyProfileFiles(this._groupDir(), model);
|
|
391
411
|
}
|
|
392
412
|
// The `version` and model `name` a model document declares, each undefined
|
|
393
413
|
// when the document has none. In a document with a YAML error, the version
|
|
@@ -520,6 +540,65 @@ function formatOverviewSidecar({ content = '', ...rest }) {
|
|
|
520
540
|
(0, documents_1.joinFrontmatter)(rest, body) :
|
|
521
541
|
body;
|
|
522
542
|
}
|
|
543
|
+
// The error checkProfileFileName throws for a profile file that breaks one of
|
|
544
|
+
// its rules.
|
|
545
|
+
class ProfileFileError extends Error {
|
|
546
|
+
}
|
|
547
|
+
exports.ProfileFileError = ProfileFileError;
|
|
548
|
+
// Throws when the profile file `file`, holding `text` for profile `name` beside
|
|
549
|
+
// the model file `modelFile`, names the same profile as a file in `seen`, which
|
|
550
|
+
// maps each profile name read before, in lower case, to its file. It also
|
|
551
|
+
// throws when the name breaks the naming rule, the file does not parse, uses
|
|
552
|
+
// the legacy `semantic_model:` wrapper, takes the name reserved for the inline
|
|
553
|
+
// bindings, or declares no name or another name. Adds the file to `seen`.
|
|
554
|
+
function checkProfileFileName(modelFile, file, name, text, seen) {
|
|
555
|
+
const badName = profileNameError(file, name);
|
|
556
|
+
if (badName)
|
|
557
|
+
throw new ProfileFileError(badName);
|
|
558
|
+
// Profile names are unique ignoring case.
|
|
559
|
+
const twin = seen.get(name.toLowerCase());
|
|
560
|
+
if (twin) {
|
|
561
|
+
throw new ProfileFileError(`Profile files '${twin}' and '${file}' name the same profile; ` +
|
|
562
|
+
`profile names are unique ignoring case.`);
|
|
563
|
+
}
|
|
564
|
+
seen.set(name.toLowerCase(), file);
|
|
565
|
+
const declared = parseProfileName(text, file, name);
|
|
566
|
+
const reserved = (n) => n?.toLowerCase() === DEFAULT_PROFILE_NAME;
|
|
567
|
+
if (reserved(name)) {
|
|
568
|
+
throw new ProfileFileError(`Profile name '${DEFAULT_PROFILE_NAME}' is reserved for the ` +
|
|
569
|
+
`inline bindings in '${modelFile}'; remove '${file}'.`);
|
|
570
|
+
}
|
|
571
|
+
if (reserved(declared)) {
|
|
572
|
+
throw new ProfileFileError(`Profile name '${DEFAULT_PROFILE_NAME}' is reserved for the ` +
|
|
573
|
+
`inline bindings in '${modelFile}'; change the name in '${file}' to ` +
|
|
574
|
+
`'${name}'.`);
|
|
575
|
+
}
|
|
576
|
+
if (declared === undefined) {
|
|
577
|
+
throw new ProfileFileError(`Profile file '${file}' declares no name; add 'name: ${name}'.`);
|
|
578
|
+
}
|
|
579
|
+
if (declared !== name) {
|
|
580
|
+
throw new ProfileFileError(`Profile file '${file}' declares name '${declared}', which does not ` +
|
|
581
|
+
`match filename suffix '${name}'.`);
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
// Push's error for the profile file `file` whose profile name `name` breaks
|
|
585
|
+
// the naming rule, or undefined. A `<model>.profile.yaml` names no profile.
|
|
586
|
+
function profileNameError(file, name) {
|
|
587
|
+
const nameless = file.match(/^(.*)\.profile\.yaml$/);
|
|
588
|
+
if (nameless && name === '') {
|
|
589
|
+
return `Profile file '${file}' names no profile; rename it ` +
|
|
590
|
+
`'${nameless[1]}.profile.<name>.yaml'.`;
|
|
591
|
+
}
|
|
592
|
+
if ((0, names_1.nameProblem)('profile', name) === undefined)
|
|
593
|
+
return undefined;
|
|
594
|
+
return `Profile file '${file}' has profile name '${name}'; a profile name ` +
|
|
595
|
+
`is a letter followed by letters, digits, underscores and hyphens.`;
|
|
596
|
+
}
|
|
597
|
+
// Push's error for the profile file `file`, which does not parse, with the
|
|
598
|
+
// parser's `message`.
|
|
599
|
+
function profileParseError(file, message) {
|
|
600
|
+
return `Profile file '${file}' does not parse: ${message}`;
|
|
601
|
+
}
|
|
523
602
|
// The `name` a profile file declares at its top level, or undefined when it
|
|
524
603
|
// declares none. A file that does not parse or that uses the legacy
|
|
525
604
|
// `semantic_model:` wrapper throws, naming the file.
|
|
@@ -529,10 +608,10 @@ function parseProfileName(text, file, name) {
|
|
|
529
608
|
doc = yaml.parse(text, yaml_options_1.YAML_OPTIONS);
|
|
530
609
|
}
|
|
531
610
|
catch (err) {
|
|
532
|
-
throw new
|
|
611
|
+
throw new ProfileFileError(profileParseError(file, err?.message ?? err));
|
|
533
612
|
}
|
|
534
613
|
if (doc?.semantic_model !== undefined) {
|
|
535
|
-
throw new
|
|
614
|
+
throw new ProfileFileError(`Profile file '${file}' uses the legacy 'semantic_model:' wrapper, ` +
|
|
536
615
|
`which is no longer supported; write its entities, relationships ` +
|
|
537
616
|
`and actions at the top level under 'name: ${name}', and move any ` +
|
|
538
617
|
`'deployment_target' to an entry under 'deployments:' in the model ` +
|
|
@@ -7,7 +7,7 @@ export interface Concept {
|
|
|
7
7
|
export declare const RESIDUE_KEY = "x-kcmd";
|
|
8
8
|
export declare const EXTRA_KEY = "extra";
|
|
9
9
|
export declare const ENTRY_SOURCE_KEYS: readonly ["title", "description", "resource"];
|
|
10
|
-
export declare const OKF_METADATA_KEYS: readonly ["tags", "status", "stale_after", "generated", "verified", "sources", "usage_window"];
|
|
10
|
+
export declare const OKF_METADATA_KEYS: readonly ["tags", "status", "superseded_by", "stale_after", "runtime", "parameters", "computation", "executor", "attester", "generated", "verified", "sources", "usage_window"];
|
|
11
11
|
export declare const KEY_ORDER: readonly string[];
|
|
12
12
|
export declare function fileContentToConcept(text: string, id: string): Concept;
|
|
13
13
|
/** Renders a concept back into file content. */
|
|
@@ -44,7 +44,13 @@ exports.ENTRY_SOURCE_KEYS = ['title', 'description', 'resource'];
|
|
|
44
44
|
exports.OKF_METADATA_KEYS = [
|
|
45
45
|
'tags',
|
|
46
46
|
'status',
|
|
47
|
+
'superseded_by',
|
|
47
48
|
'stale_after',
|
|
49
|
+
'runtime',
|
|
50
|
+
'parameters',
|
|
51
|
+
'computation',
|
|
52
|
+
'executor',
|
|
53
|
+
'attester',
|
|
48
54
|
'generated',
|
|
49
55
|
'verified',
|
|
50
56
|
'sources',
|
|
@@ -2,6 +2,8 @@ import * as dataplex from '../gcp/dataplex';
|
|
|
2
2
|
import { Concept } from './concept';
|
|
3
3
|
export declare class ConceptError extends Error {
|
|
4
4
|
}
|
|
5
|
+
/** Valid Dataplex `query-template` aspect `parameters[].type` enum values. */
|
|
6
|
+
export declare const VALID_QUERY_TEMPLATE_PARAM_TYPES: Set<string>;
|
|
5
7
|
/** Maps OKF human-readable `type` labels to Dataplex `document.type` slugs. */
|
|
6
8
|
export declare function normalizeOkfDocType(rawType: string): string;
|
|
7
9
|
export declare function entryToConcept(entry: dataplex.Entry, id: string): Concept;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// Bidirectional conversion between a Dataplex `Entry` and an OKF `Concept`.
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.ConceptError = void 0;
|
|
4
|
+
exports.VALID_QUERY_TEMPLATE_PARAM_TYPES = exports.ConceptError = void 0;
|
|
5
5
|
exports.normalizeOkfDocType = normalizeOkfDocType;
|
|
6
6
|
exports.entryToConcept = entryToConcept;
|
|
7
7
|
exports.conceptToEntry = conceptToEntry;
|
|
@@ -27,6 +27,18 @@ function headingTitle(body) {
|
|
|
27
27
|
}
|
|
28
28
|
return undefined;
|
|
29
29
|
}
|
|
30
|
+
/** Valid Dataplex `query-template` aspect `parameters[].type` enum values. */
|
|
31
|
+
exports.VALID_QUERY_TEMPLATE_PARAM_TYPES = new Set([
|
|
32
|
+
'BOOLEAN',
|
|
33
|
+
'NUMBER',
|
|
34
|
+
'STRING',
|
|
35
|
+
'BYTES',
|
|
36
|
+
'DATETIME',
|
|
37
|
+
'TIMESTAMP',
|
|
38
|
+
'GEOSPATIAL',
|
|
39
|
+
'STRUCT',
|
|
40
|
+
'RANGE',
|
|
41
|
+
]);
|
|
30
42
|
/** Maps OKF human-readable `type` labels to Dataplex `document.type` slugs. */
|
|
31
43
|
function normalizeOkfDocType(rawType) {
|
|
32
44
|
const norm = rawType
|
|
@@ -34,6 +46,12 @@ function normalizeOkfDocType(rawType) {
|
|
|
34
46
|
.toLowerCase()
|
|
35
47
|
.replace(/[^a-z0-9]+/g, '-')
|
|
36
48
|
.replace(/^-|-$/g, '');
|
|
49
|
+
if (norm === 'context-set' || norm === 'knowledge-base-root') {
|
|
50
|
+
return 'context-set';
|
|
51
|
+
}
|
|
52
|
+
if (norm === 'knowledge-base-index' || norm === 'index') {
|
|
53
|
+
return 'knowledge-base-index';
|
|
54
|
+
}
|
|
37
55
|
if (norm === 'table' ||
|
|
38
56
|
norm === 'bigquery-table' ||
|
|
39
57
|
norm === 'table-context') {
|
|
@@ -218,6 +236,10 @@ function conceptToEntry(concept, entryGroup) {
|
|
|
218
236
|
if (stripped.length) {
|
|
219
237
|
console.warn(`[kcmd] ${concept.id}: stripping unsupported parameter key(s) ${stripped.join(', ')} on parameters[${idx}]`);
|
|
220
238
|
}
|
|
239
|
+
const paramType = text(cleaned['type']);
|
|
240
|
+
if (paramType && !exports.VALID_QUERY_TEMPLATE_PARAM_TYPES.has(paramType)) {
|
|
241
|
+
throw new ConceptError(`${concept.id}: ${catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY}.parameters[${idx}].type '${paramType}' is invalid; must be one of ${[...exports.VALID_QUERY_TEMPLATE_PARAM_TYPES].join(', ')}`);
|
|
242
|
+
}
|
|
221
243
|
return cleaned;
|
|
222
244
|
});
|
|
223
245
|
}
|
|
@@ -234,7 +256,12 @@ function conceptToEntry(concept, entryGroup) {
|
|
|
234
256
|
facetKey in aspects &&
|
|
235
257
|
(facetKey === catalog_residue_1.DOCUMENT_ASPECT_KEY ||
|
|
236
258
|
normalizeOkfDocType(conceptType) !== shortEntryType)) {
|
|
237
|
-
|
|
259
|
+
const resolvedType = facetKey === catalog_residue_1.DOCUMENT_ASPECT_KEY
|
|
260
|
+
? normalizeOkfDocType(conceptType)
|
|
261
|
+
: conceptType;
|
|
262
|
+
aspects[facetKey] = {
|
|
263
|
+
data: { ...aspects[facetKey].data, type: resolvedType },
|
|
264
|
+
};
|
|
238
265
|
}
|
|
239
266
|
else if (conceptType &&
|
|
240
267
|
!residueEntryType &&
|
|
@@ -76,18 +76,13 @@ async function initOkfWorkspace(ctx, manifest, options = {}) {
|
|
|
76
76
|
const prior = fs.existsSync('catalog.yaml')
|
|
77
77
|
? fs.readFileSync('catalog.yaml', 'utf8')
|
|
78
78
|
: undefined;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
else {
|
|
83
|
-
const scopeType = manifest.source.type || 'entryGroup';
|
|
84
|
-
fs.writeFileSync('catalog.yaml', yaml.stringify({ scope: `${scopeType}.${scopeName}` }), 'utf8');
|
|
85
|
-
}
|
|
79
|
+
const scopeType = manifest.source.type || 'entryGroup';
|
|
80
|
+
fs.writeFileSync('catalog.yaml', yaml.stringify({ scope: `${scopeType}.${scopeName}`, layout: 'okf' }), 'utf8');
|
|
86
81
|
const scope = scopeOf(fs.readFileSync('catalog.yaml', 'utf8'));
|
|
87
82
|
if (prior !== undefined && scope !== undefined && scopeOf(prior) === scope) {
|
|
88
83
|
fs.writeFileSync('catalog.yaml', prior);
|
|
84
|
+
(0, manifest_1.writeWorkspaceLayout)('catalog.yaml', 'okf');
|
|
89
85
|
}
|
|
90
|
-
(0, manifest_1.writeWorkspaceLayout)('catalog.yaml', 'okf');
|
|
91
86
|
console.log(fs.readFileSync('catalog.yaml', 'utf8').trimEnd());
|
|
92
87
|
const remoteExists = manifest.okfRemoteExists === true;
|
|
93
88
|
const shouldPull = (options.pull && remoteExists) ||
|
|
@@ -108,8 +103,11 @@ async function initOkfWorkspace(ctx, manifest, options = {}) {
|
|
|
108
103
|
fs.mkdirSync(okf_1.BUNDLE_DIR, { recursive: true });
|
|
109
104
|
const indexPath = path.join(okf_1.BUNDLE_DIR, 'index.md');
|
|
110
105
|
const agentsPath = path.join(okf_1.BUNDLE_DIR, 'agents.md');
|
|
111
|
-
|
|
112
|
-
|
|
106
|
+
const hasConceptFiles = fs
|
|
107
|
+
.readdirSync(okf_1.BUNDLE_DIR)
|
|
108
|
+
.some((f) => f.endsWith('.md'));
|
|
109
|
+
if (!fs.existsSync(indexPath) && !fs.existsSync(agentsPath) && !hasConceptFiles) {
|
|
110
|
+
fs.writeFileSync(indexPath, `---\nokf_version: "0.3"\n---\n# ${groupId}\n`, 'utf8');
|
|
113
111
|
}
|
|
114
112
|
return 0;
|
|
115
113
|
}
|