@google-cloud/kcmd 0.0.0-stage → 0.1.0

package/build/ts/kcmd/okf/concept_to_entry.js

@@ -0,0 +1,254 @@
+"use strict";
+// Bidirectional conversion between a Dataplex `Entry` and an OKF `Concept`.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConceptError = void 0;
+exports.normalizeOkfDocType = normalizeOkfDocType;
+exports.entryToConcept = entryToConcept;
+exports.conceptToEntry = conceptToEntry;
+const catalog_residue_1 = require("./catalog_residue");
+const concept_1 = require("./concept");
+const okf_metadata_1 = require("./okf_metadata");
+class ConceptError extends Error {
+}
+exports.ConceptError = ConceptError;
+function mapping(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : {};
+}
+function text(value) {
+    return typeof value === 'string' && value !== '' ? value : undefined;
+}
+function headingTitle(body) {
+    for (const line of body.split(/\r?\n/)) {
+        const m = line.match(/^#\s+(.+?)\s*$/);
+        if (m && m[1])
+            return m[1];
+    }
+    return undefined;
+}
+/** Maps OKF human-readable `type` labels to Dataplex `document.type` slugs. */
+function normalizeOkfDocType(rawType) {
+    const norm = rawType
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '');
+    if (norm === 'table' ||
+        norm === 'bigquery-table' ||
+        norm === 'table-context') {
+        return 'table-context';
+    }
+    return norm || 'document';
+}
+function bodyContent(body, hadTrailingNewline) {
+    if (hadTrailingNewline)
+        return body;
+    return body.endsWith('\n') ? body.slice(0, -1) : body;
+}
+function conceptTypeOf(entry) {
+    const key = (0, catalog_residue_1.typeFacetKey)(entry.entryType);
+    if (key) {
+        const label = entry.aspects?.[key]?.data?.['type'];
+        if (typeof label === 'string' && label)
+            return label;
+    }
+    return entry.entryType?.split('/').pop() || 'concept';
+}
+function entryToConcept(entry, id) {
+    const source = entry.entrySource ?? {};
+    const frontmatter = { type: conceptTypeOf(entry) };
+    if (source.displayName)
+        frontmatter['title'] = source.displayName;
+    if (source.description)
+        frontmatter['description'] = source.description;
+    if (source.resource)
+        frontmatter['resource'] = source.resource;
+    const overview = entry.aspects?.[catalog_residue_1.OVERVIEW_ASPECT_KEY];
+    const content = typeof overview?.data?.['content'] === 'string'
+        ? overview.data['content']
+        : '';
+    const { body, metadata } = (0, okf_metadata_1.splitOkfMetadata)(content);
+    Object.assign(frontmatter, metadata);
+    const residue = (0, catalog_residue_1.buildResidue)(entry, catalog_residue_1.OVERVIEW_ASPECT_KEY, body);
+    if (residue)
+        frontmatter[concept_1.RESIDUE_KEY] = residue;
+    (0, catalog_residue_1.assertResidueOnly)(frontmatter);
+    return { id, frontmatter, body };
+}
+function conceptToEntry(concept, entryGroup) {
+    const frontmatter = concept.frontmatter;
+    (0, catalog_residue_1.assertResidueOnly)(frontmatter);
+    const residue = mapping(frontmatter[concept_1.RESIDUE_KEY]);
+    const extra = mapping(frontmatter[concept_1.EXTRA_KEY]);
+    const extraResource = mapping(extra['resource']);
+    const residueEntryType = text(residue['entryType']);
+    const extraEntryType = text(extra['type']);
+    const rawExplicitType = residueEntryType ?? extraEntryType;
+    const hasQueryTemplateAspect = catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY in mapping(extra['aspects']) ||
+        catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY in mapping(residue['aspects']);
+    const inferredQueryTemplateType = !rawExplicitType &&
+        hasQueryTemplateAspect &&
+        normalizeOkfDocType(text(frontmatter['type']) ?? '') === 'query-template'
+        ? catalog_residue_1.QUERY_TEMPLATE_ENTRY_TYPE
+        : undefined;
+    const normalizedExplicitType = rawExplicitType ?? inferredQueryTemplateType;
+    const isQueryTemplate = Boolean(normalizedExplicitType?.endsWith('/entryTypes/query-template'));
+    const defaultedEntryType = !normalizedExplicitType;
+    const isRootIndex = concept.id === 'index';
+    const isDirIndex = concept.id.endsWith('/index');
+    let conceptType = text(frontmatter['type']);
+    if (!conceptType && defaultedEntryType) {
+        if (isRootIndex) {
+            conceptType = 'knowledge-base';
+        }
+        else if (isDirIndex) {
+            conceptType = 'knowledge-base-index';
+        }
+        else {
+            throw new ConceptError(`${concept.id}: no ${concept_1.RESIDUE_KEY}.entryType and no frontmatter 'type', ` +
+                `so the catalog entry type is unknown`);
+        }
+    }
+    const entryType = normalizedExplicitType ?? catalog_residue_1.DEFAULT_DOCUMENT_ENTRY_TYPE;
+    let resolvedId = text(residue['entryId']) ?? text(extra['entry_id']) ?? concept.id;
+    if (defaultedEntryType &&
+        !text(residue['entryId']) &&
+        !text(extra['entry_id'])) {
+        if (isRootIndex) {
+            resolvedId = 'agents';
+        }
+        else if (isDirIndex) {
+            resolvedId = concept.id.slice(0, -'/index'.length);
+        }
+    }
+    const entry = {
+        name: `${entryGroup}/entries/${resolvedId}`,
+        entryType,
+    };
+    const explicitParent = text(residue['parentEntry']) ?? text(extraResource['parent']);
+    if (explicitParent) {
+        entry.parentEntry = explicitParent;
+    }
+    else if (defaultedEntryType) {
+        if (isRootIndex) {
+            // Root `index.md` (`agents`) is the top-level entry; no parentEntry.
+        }
+        else if (isDirIndex) {
+            const slash = resolvedId.lastIndexOf('/');
+            entry.parentEntry =
+                slash !== -1
+                    ? `${entryGroup}/entries/${resolvedId.slice(0, slash)}`
+                    : `${entryGroup}/entries/agents`;
+        }
+        else {
+            const slash = resolvedId.lastIndexOf('/');
+            if (slash !== -1) {
+                entry.parentEntry = `${entryGroup}/entries/${resolvedId.slice(0, slash)}`;
+            }
+        }
+    }
+    const source = { ...mapping(residue['entrySource']) };
+    for (const key of ['system', 'platform', 'location', 'ancestors', 'labels']) {
+        if (source[key] === undefined && extraResource[key] !== undefined) {
+            source[key] = extraResource[key];
+        }
+    }
+    const displayName = text(frontmatter['title']) ??
+        (defaultedEntryType || isQueryTemplate
+            ? headingTitle(concept.body)
+            : undefined);
+    const description = text(frontmatter['description']);
+    const resource = text(frontmatter['resource']);
+    if (displayName)
+        source['displayName'] = displayName;
+    if (description)
+        source['description'] = description;
+    if (resource)
+        source['resource'] = resource;
+    if (Object.keys(source).length) {
+        entry.entrySource = source;
+    }
+    const aspects = {};
+    for (const [key, data] of Object.entries(mapping(extra['aspects']))) {
+        const mapped = mapping(data);
+        const inner = Object.keys(mapped).length === 1 && 'data' in mapped
+            ? mapping(mapped['data'])
+            : mapped;
+        aspects[key] = { data: { ...inner } };
+    }
+    for (const [key, data] of Object.entries(mapping(residue['aspects']))) {
+        aspects[key] = { data: { ...mapping(data) } };
+    }
+    // A type with no stashed aspect was read off the entry type, not an aspect,
+    // unless the entry came from a pure OKF file (defaultedEntryType) or `extra`.
+    const facetKey = (0, catalog_residue_1.typeFacetKey)(entryType);
+    const shortEntryType = entryType.split('/').pop();
+    if (isQueryTemplate) {
+        const qtAspect = aspects[catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY];
+        if (!qtAspect) {
+            throw new ConceptError(`${concept.id}: query-template entry requires aspect '${catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY}'`);
+        }
+        const qtData = qtAspect.data;
+        for (const reqField of ['query', 'nlQuery', 'intent', 'sourceType']) {
+            if (!text(qtData[reqField])) {
+                throw new ConceptError(`${concept.id}: ${catalog_residue_1.QUERY_TEMPLATE_ASPECT_KEY}.${reqField} is required`);
+            }
+        }
+        if (Array.isArray(qtData['parameters'])) {
+            const allowedParamKeys = new Set([
+                'name',
+                'type',
+                'description',
+                'sampleValues',
+                'targets',
+            ]);
+            qtData['parameters'] = qtData['parameters'].map((rawParam, idx) => {
+                const param = mapping(rawParam);
+                const cleaned = {};
+                const stripped = [];
+                for (const [k, v] of Object.entries(param)) {
+                    if (allowedParamKeys.has(k)) {
+                        cleaned[k] = v;
+                    }
+                    else {
+                        stripped.push(k);
+                    }
+                }
+                if (stripped.length) {
+                    console.warn(`[kcmd] ${concept.id}: stripping unsupported parameter key(s) ${stripped.join(', ')} on parameters[${idx}]`);
+                }
+                return cleaned;
+            });
+        }
+        delete aspects[catalog_residue_1.DOCUMENT_ASPECT_KEY];
+    }
+    else if (defaultedEntryType) {
+        const docType = normalizeOkfDocType(conceptType ?? 'document');
+        aspects[catalog_residue_1.DOCUMENT_ASPECT_KEY] = {
+            data: { type: docType, ...(aspects[catalog_residue_1.DOCUMENT_ASPECT_KEY]?.data ?? {}) },
+        };
+    }
+    else if (conceptType &&
+        facetKey &&
+        facetKey in aspects &&
+        (facetKey === catalog_residue_1.DOCUMENT_ASPECT_KEY ||
+            normalizeOkfDocType(conceptType) !== shortEntryType)) {
+        aspects[facetKey] = { data: { type: conceptType, ...aspects[facetKey].data } };
+    }
+    else if (conceptType &&
+        !residueEntryType &&
+        extraEntryType &&
+        facetKey === catalog_residue_1.DOCUMENT_ASPECT_KEY &&
+        !(facetKey in aspects)) {
+        aspects[facetKey] = { data: { type: normalizeOkfDocType(conceptType) } };
+    }
+    const content = (0, okf_metadata_1.appendOkfMetadata)(bodyContent(concept.body, residue['trailingNewline'] !== false), frontmatter);
+    if (content) {
+        aspects[catalog_residue_1.OVERVIEW_ASPECT_KEY] = { data: { content, contentType: 'MARKDOWN' } };
+    }
+    if (Object.keys(aspects).length) {
+        entry.aspects = aspects;
+    }
+    return entry;
+}

package/build/ts/kcmd/okf/init_okf_workspace.d.ts

@@ -0,0 +1,15 @@
+import * as gcp from '../gcp';
+import { CatalogManifest } from '../manifest';
+/** Options forwarded from `kcmd init`. */
+export interface InitOkfWorkspaceOptions {
+    /** Pull the context set straight after scaffolding. */
+    pull?: boolean;
+}
+/**
+ * Scaffolds an OKF workspace for the given scope: `catalog.yaml` at the root,
+ * and beside it a `bundle/` directory (pulling existing concepts if the
+ * EntryGroup exists remotely and `--pull` is requested or `okfRemoteExists` is set).
+ *
+ * @returns a process exit code: 0 on success, non-zero on failure.
+ */
+export declare function initOkfWorkspace(ctx: gcp.ApiContext, manifest: CatalogManifest, options?: InitOkfWorkspaceOptions): Promise<number>;

package/build/ts/kcmd/okf/init_okf_workspace.js

@@ -0,0 +1,115 @@
+"use strict";
+// The `kcmd init --format=okf` path: scaffold an OKF workspace.
+//
+// Writes `catalog.yaml` (`scope: entryGroup.<project>.<location>.<id>` and
+// `layout: okf`, which `kcmd pull` and `kcmd push` read to take the OKF path)
+// and a `bundle/index.md` root skeleton. If `--pull` is set (or the entry group
+// already exists remotely via `init`), pulls the remote entries into `bundle/`.
+// When the entry group does not exist yet, it is left uncreated until `kcmd push`
+// (so plan/review gates hold before any remote mutation).
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initOkfWorkspace = initOkfWorkspace;
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const yaml = __importStar(require("yaml"));
+const dataplex = __importStar(require("../gcp/dataplex"));
+const okf_1 = require("../layouts/okf");
+const manifest_1 = require("../manifest");
+const snapshot_1 = require("../snapshot");
+const sync_1 = require("../sync");
+function scopeOf(text) {
+    try {
+        return JSON.stringify(yaml.parse(text)?.scope);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Scaffolds an OKF workspace for the given scope: `catalog.yaml` at the root,
+ * and beside it a `bundle/` directory (pulling existing concepts if the
+ * EntryGroup exists remotely and `--pull` is requested or `okfRemoteExists` is set).
+ *
+ * @returns a process exit code: 0 on success, non-zero on failure.
+ */
+async function initOkfWorkspace(ctx, manifest, options = {}) {
+    const scopeName = manifest?.source?.name ?? '';
+    const parts = scopeName.split('.');
+    if (parts.length !== 3 || parts.some((p) => !p)) {
+        console.error(`Error: EntryGroup must be in format <projectId>.<locationId>.<entryGroupId>: '${scopeName}'`);
+        return 1;
+    }
+    const [, , groupId] = parts;
+    const prior = fs.existsSync('catalog.yaml')
+        ? fs.readFileSync('catalog.yaml', 'utf8')
+        : undefined;
+    if (typeof manifest.save === 'function') {
+        manifest.save('catalog.yaml');
+    }
+    else {
+        const scopeType = manifest.source.type || 'entryGroup';
+        fs.writeFileSync('catalog.yaml', yaml.stringify({ scope: `${scopeType}.${scopeName}` }), 'utf8');
+    }
+    const scope = scopeOf(fs.readFileSync('catalog.yaml', 'utf8'));
+    if (prior !== undefined && scope !== undefined && scopeOf(prior) === scope) {
+        fs.writeFileSync('catalog.yaml', prior);
+    }
+    (0, manifest_1.writeWorkspaceLayout)('catalog.yaml', 'okf');
+    console.log(fs.readFileSync('catalog.yaml', 'utf8').trimEnd());
+    const remoteExists = manifest.okfRemoteExists === true;
+    const shouldPull = (options.pull && remoteExists) ||
+        (remoteExists && options.pull === undefined);
+    if (shouldPull) {
+        const snapshot = await snapshot_1.CatalogSnapshot.fromPath('.', ctx);
+        const sync = new sync_1.CatalogSync(new dataplex.CatalogClient(ctx), snapshot);
+        const result = await sync.pull({ dryRun: false });
+        if (!result.success) {
+            if (result.details)
+                console.error(`Error: ${result.details}`);
+            return 1;
+        }
+    }
+    else {
+        console.log(`Initialized OKF workspace for ${scopeName} in ${okf_1.BUNDLE_DIR}/ (entry group does not exist yet; \`kcmd push\` creates it).`);
+    }
+    fs.mkdirSync(okf_1.BUNDLE_DIR, { recursive: true });
+    const indexPath = path.join(okf_1.BUNDLE_DIR, 'index.md');
+    const agentsPath = path.join(okf_1.BUNDLE_DIR, 'agents.md');
+    if (!fs.existsSync(indexPath) && !fs.existsSync(agentsPath)) {
+        fs.writeFileSync(indexPath, `---\nokf_version: "0.2"\n---\n# ${groupId}\n`, 'utf8');
+    }
+    return 0;
+}

package/build/ts/kcmd/okf/links.d.ts

@@ -0,0 +1,23 @@
+import * as gcp from '../gcp';
+import { Concept } from './concept';
+export declare const LINKS_FILE = "_links.yaml";
+export interface BundleLinksResult {
+    /** True when `_links.yaml` exists or a concept declares frontmatter links. */
+    hasLinksConfig: boolean;
+    /** Deduplicated Dataplex EntryLinks ready for push. */
+    links: gcp.EntryLink[];
+}
+/** Reads `bundle/_links.yaml` and any frontmatter `links` from `concepts`. */
+export declare function readBundleLinks(root: string, entryGroup: string, concepts?: Concept[]): BundleLinksResult;
+/** Formats Dataplex `EntryLink`s into canonical `bundle/_links.yaml` content. */
+export declare function formatLinksYaml(links: gcp.EntryLink[], entryGroup: string): string;
+/** Extracts the bare `entryLinkId` from an `EntryLink.name`. */
+export declare function entryLinkId(link: gcp.EntryLink): string;
+/** Deterministic deduplication key across link type and endpoints. */
+export declare function linkDedupKey(link: gcp.EntryLink): string;
+/** True when `existing` differs from `desired` in immutable type or references. */
+export declare function isStaleEntryLink(existing: gcp.EntryLink, desired: gcp.EntryLink): boolean;
+export declare function resolveEntryLinkType(rawType: string): string;
+export declare function shortenLinkType(entryLinkType: string): string;
+export declare function resolveEntryRefName(rawName: string, entryGroup: string): string;
+export declare function shortenEntryName(fullEntryName: string, entryGroup: string): string;