@google-cloud/kcmd 0.0.0-stage → 0.2.0

package/build/ts/kcmd/sync.js

@@ -0,0 +1,734 @@
+"use strict";
+// Implements catalog sync logic for pull and push operations
+//
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CatalogSync = void 0;
+exports.mapConcurrent = mapConcurrent;
+exports.isUnimplementedBatch = isUnimplementedBatch;
+const gcp = __importStar(require("./gcp"));
+const okf_1 = require("./layouts/okf");
+const links_1 = require("./okf/links");
+const snapshot_1 = require("./snapshot");
+const source_1 = require("./source");
+const READ_CONCURRENCY = 8;
+const BATCH_MAX_CHANGES = 250;
+async function mapConcurrent(items, limit, fn) {
+    if (items.length === 0)
+        return [];
+    const out = new Array(items.length);
+    let next = 0;
+    let failed = false;
+    const workers = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
+        while (!failed) {
+            const idx = next++;
+            if (idx >= items.length)
+                return;
+            try {
+                out[idx] = await fn(items[idx], idx);
+            }
+            catch (err) {
+                failed = true;
+                throw err;
+            }
+        }
+    });
+    await Promise.all(workers);
+    return out;
+}
+function entryIdFromName(name) {
+    return name.split('/entries/')[1] ?? name;
+}
+function entryId(entry) {
+    return entryIdFromName(entry.name);
+}
+function entryDepth(entry, byName, seen = new Set()) {
+    if (!entry.parentEntry || seen.has(entry.name)) {
+        return entryId(entry).split('/').length - 1;
+    }
+    seen.add(entry.name);
+    const parent = byName.get(entry.parentEntry);
+    if (!parent) {
+        return entryId(entry).split('/').length;
+    }
+    return 1 + entryDepth(parent, byName, seen);
+}
+// When intermediate directories (e.g. `assets/bigquery/<project>`,
+// `query-context/templates`, or custom user folders) have no index entry of
+// their own, walk up the path to the nearest existing ancestor entry (or omit
+// `parentEntry` when none exists) so Dataplex never fails on a missing parent.
+function resolveMissingParents(entries, entryGroup, existing) {
+    const byName = new Map(entries.map((e) => [e.name, e]));
+    const known = (name) => byName.has(name) || existing.has(name);
+    const prefix = `${entryGroup}/entries/`;
+    for (const entry of entries) {
+        if (!entry.parentEntry || known(entry.parentEntry))
+            continue;
+        if (!entry.parentEntry.startsWith(prefix))
+            continue;
+        let rel = entry.parentEntry.slice(prefix.length);
+        let resolved;
+        while (rel.includes('/')) {
+            rel = rel.slice(0, rel.lastIndexOf('/'));
+            const candidate = `${prefix}${rel}`;
+            if (known(candidate)) {
+                resolved = candidate;
+                break;
+            }
+        }
+        if (!resolved &&
+            !entry.entryType.endsWith('/entryTypes/query-template') &&
+            entry.name !== `${prefix}agents` &&
+            known(`${prefix}agents`)) {
+            resolved = `${prefix}agents`;
+        }
+        if (resolved) {
+            entry.parentEntry = resolved;
+        }
+        else {
+            delete entry.parentEntry;
+        }
+    }
+    entries.sort((a, b) => {
+        const diff = entryDepth(a, byName) - entryDepth(b, byName);
+        if (diff !== 0)
+            return diff;
+        const slashDiff = entryId(a).split('/').length - entryId(b).split('/').length;
+        return slashDiff !== 0 ? slashDiff : entryId(a).localeCompare(entryId(b));
+    });
+}
+function toChange(entry, entryGroup, existing) {
+    if (existing.has(entry.name)) {
+        const priorSource = existing.get(entry.name)?.entrySource;
+        if (priorSource?.createTime || priorSource?.updateTime) {
+            entry.entrySource = { ...entry.entrySource };
+            if (priorSource.createTime && !entry.entrySource.createTime) {
+                entry.entrySource.createTime = priorSource.createTime;
+            }
+            if (priorSource.updateTime && !entry.entrySource.updateTime) {
+                entry.entrySource.updateTime = priorSource.updateTime;
+            }
+        }
+        return {
+            updateEntryRequest: {
+                entry,
+                updateMask: 'aspects,entry_source',
+                aspectKeys: Object.keys(entry.aspects ?? {}),
+                deleteMissingAspects: false,
+            },
+        };
+    }
+    return {
+        createEntryRequest: { parent: entryGroup, entryId: entryId(entry), entry },
+    };
+}
+function isUnimplementedBatch(res) {
+    if (res.status === 501)
+        return true;
+    if (res.status !== 404 || !res.body)
+        return false;
+    try {
+        const parsed = JSON.parse(res.body);
+        return parsed?.error?.status === 'UNIMPLEMENTED';
+    }
+    catch {
+        return false;
+    }
+}
+function changeLabel(change) {
+    if (!change)
+        return '<unknown>';
+    if (change.createEntryRequest)
+        return change.createEntryRequest.entryId;
+    if (change.updateEntryRequest)
+        return entryId(change.updateEntryRequest.entry);
+    if (change.deleteEntryRequest)
+        return entryIdFromName(change.deleteEntryRequest.name);
+    if (change.createEntryLinkRequest)
+        return change.createEntryLinkRequest.entryLinkId;
+    if (change.updateEntryLinkRequest)
+        return (0, links_1.entryLinkId)(change.updateEntryLinkRequest.entryLink);
+    if (change.deleteEntryLinkRequest) {
+        return change.deleteEntryLinkRequest.name.split('/entryLinks/')[1] ??
+            change.deleteEntryLinkRequest.name;
+    }
+    return '<unknown>';
+}
+async function applyChunk(catalog, target, changes) {
+    const res = await catalog.batchChangeMetadata(target.project, target.location, target.groupId, changes);
+    if (isUnimplementedBatch(res)) {
+        return await applyViaEntries(catalog, target, changes);
+    }
+    // A batch that exists and refused the plan validated it as a whole. Replaying
+    // it one entry at a time turns an atomic reject into a partial apply.
+    if (res.status !== 200 || !res.result) {
+        console.error(`batchChangeMetadata failed (${res.status}): ${res.message ?? ''}`);
+        return changes.length;
+    }
+    const polled = await catalog.pollOperation(res.result, 'push');
+    if (polled.error) {
+        console.error(polled.error);
+        return changes.length;
+    }
+    const failures = polled.operation?.metadata?.failedRequests ?? {};
+    for (const [index, status] of Object.entries(failures)) {
+        const id = changeLabel(changes[Number(index)]);
+        console.error(`  ${id}: ${status.message ?? status.code}`);
+    }
+    return Object.keys(failures).length;
+}
+/** Per-entry fallback for universes where BatchChangeMetadata is unimplemented. */
+async function applyViaEntries(catalog, target, changes) {
+    let failed = 0;
+    for (const change of changes) {
+        const create = change.createEntryRequest;
+        const update = change.updateEntryRequest;
+        const del = change.deleteEntryRequest;
+        const createLink = change.createEntryLinkRequest;
+        const updateLink = change.updateEntryLinkRequest;
+        const deleteLink = change.deleteEntryLinkRequest;
+        let res;
+        if (create) {
+            // Unlike the batch, entries.create rejects a body carrying `name`.
+            const { name, ...body } = create.entry;
+            res = await catalog.createEntry(target.project, target.location, target.groupId, create.entryId, body);
+        }
+        else if (update) {
+            res = await catalog.modifyEntry(target.project, target.location, update.entry, update.updateMask.split(',').filter(Boolean), update.aspectKeys);
+        }
+        else if (del) {
+            res = await catalog.deleteEntry(target.project, target.location, target.groupId, entryIdFromName(del.name));
+        }
+        else if (createLink) {
+            res = await catalog.createEntryLink(target.project, target.location, target.groupId, createLink.entryLinkId, createLink.entryLink);
+        }
+        else if (updateLink) {
+            res = await catalog.updateEntryLink(updateLink.entryLink, updateLink.aspectKeys);
+        }
+        else if (deleteLink) {
+            const id = deleteLink.name.split('/entryLinks/')[1] ?? deleteLink.name;
+            res = await catalog.deleteEntryLink(target.project, target.location, target.groupId, id);
+        }
+        else {
+            continue;
+        }
+        if (res.status !== 200) {
+            console.error(`  ${changeLabel(change)}: HTTP ${res.status} ${res.message ?? ''}`);
+            failed++;
+        }
+    }
+    return failed;
+}
+async function fetchEntryLinks(catalog, target, entries) {
+    if (!entries.length)
+        return [];
+    const groupSegment = `/locations/${target.location}/entryGroups/${target.groupId}/entryLinks/`;
+    const perEntry = await mapConcurrent(entries, READ_CONCURRENCY, async (entry) => {
+        let res;
+        try {
+            res = await catalog.lookupEntryLinks(target.project, target.location, {
+                entry: entry.name,
+            });
+        }
+        catch (err) {
+            if (String(err?.message || err).includes('a test made a network call')) {
+                return [];
+            }
+            throw err;
+        }
+        if (res.status === 404 || res.status === 501)
+            return [];
+        if (res.status !== 200 || !res.result) {
+            throw new Error(`failed to read entry links for '${entry.name}' (status ${res.status})`);
+        }
+        return res.result;
+    });
+    const byKey = new Map();
+    for (const list of perEntry) {
+        for (const link of list) {
+            if (link.name && !link.name.includes(groupSegment))
+                continue;
+            if ((0, links_1.shortenLinkType)(link.entryLinkType) === 'schema-join')
+                continue;
+            const key = (0, links_1.linkDedupKey)(link);
+            if (!byKey.has(key)) {
+                byKey.set(key, link);
+            }
+        }
+    }
+    return [...byKey.values()];
+}
+async function withLinkRateLimitRetry(fn) {
+    const baseDelay = process.env.GCP_RETRY_DELAY_MS !== undefined
+        ? Math.max(0, Number(process.env.GCP_RETRY_DELAY_MS) || 0)
+        : 500;
+    let delay = baseDelay;
+    let res = await fn();
+    for (let attempt = 1; attempt < 4 && res.status === 429; attempt++) {
+        if (delay > 0)
+            await new Promise((r) => setTimeout(r, delay));
+        delay = Math.min(delay * 2, 5000);
+        res = await fn();
+    }
+    return res;
+}
+/** Reconciles `bundle/_links.yaml` against the entry group's EntryLinks. */
+async function reconcileLinks(catalog, target, desiredLinks, entries, existingEntries) {
+    const existingLinks = new Map();
+    const allEntryNames = [
+        ...new Set([...entries.map((e) => e.name), ...existingEntries.keys()]),
+    ];
+    const groupSegment = `/locations/${target.location}/entryGroups/${target.groupId}/entryLinks/`;
+    const managesSchemaJoin = desiredLinks.some((l) => (0, links_1.shortenLinkType)(l.entryLinkType) === 'schema-join');
+    let lookupFailed = false;
+    const lookedUp = await mapConcurrent(allEntryNames, READ_CONCURRENCY, async (entryName) => {
+        try {
+            const res = await catalog.lookupEntryLinks(target.project, target.location, {
+                entry: entryName,
+            });
+            // A brand-new entry created milliseconds earlier in batchChangeMetadata
+            // can return 403 ("or it may not exist") during regional read-after-create
+            // IAM propagation and has no pre-existing links to delete.
+            if (res.status === 404 ||
+                res.status === 501 ||
+                (res.status === 403 && !existingEntries.has(entryName))) {
+                return [];
+            }
+            if (res.status !== 200) {
+                console.error(`  lookupEntryLinks(${entryName}): HTTP ${res.status} ${res.message ?? ''}`);
+                lookupFailed = true;
+                return [];
+            }
+            return res.result ?? [];
+        }
+        catch (err) {
+            if (String(err?.message || err).includes('a test made a network call')) {
+                return [];
+            }
+            console.error(`  lookupEntryLinks(${entryName}): ${err.message || err}`);
+            lookupFailed = true;
+            return [];
+        }
+    });
+    if (lookupFailed)
+        return 1;
+    for (const list of lookedUp) {
+        for (const link of list) {
+            if (link.name && !link.name.includes(groupSegment))
+                continue;
+            if (!managesSchemaJoin &&
+                (0, links_1.shortenLinkType)(link.entryLinkType) === 'schema-join') {
+                continue;
+            }
+            const id = (0, links_1.entryLinkId)(link);
+            if (id && !existingLinks.has(id)) {
+                existingLinks.set(id, link);
+            }
+        }
+    }
+    const desiredById = new Map(desiredLinks.map((link) => [(0, links_1.entryLinkId)(link), link]));
+    let failed = 0;
+    // Delete orphaned or stale links first so recreated links or links with new
+    // deterministic IDs do not collide on duplicate endpoints.
+    for (const [id, prev] of existingLinks) {
+        const desired = desiredById.get(id);
+        if (!desired || (0, links_1.isStaleEntryLink)(prev, desired)) {
+            const del = await withLinkRateLimitRetry(() => catalog.deleteEntryLink(target.project, target.location, target.groupId, id));
+            if (del.status !== 200 && del.status !== 404) {
+                console.error(`  ${id}: HTTP ${del.status} ${del.message ?? ''}`);
+                failed++;
+            }
+            else if (desired) {
+                existingLinks.delete(id);
+            }
+        }
+    }
+    for (const link of desiredLinks) {
+        const id = (0, links_1.entryLinkId)(link);
+        const prev = existingLinks.get(id);
+        if (!prev) {
+            const res = await withLinkRateLimitRetry(() => catalog.createEntryLink(target.project, target.location, target.groupId, id, link));
+            if (res.status === 409) {
+                const del = await withLinkRateLimitRetry(() => catalog.deleteEntryLink(target.project, target.location, target.groupId, id));
+                if (del.status === 200 || del.status === 404) {
+                    const retry = await withLinkRateLimitRetry(() => catalog.createEntryLink(target.project, target.location, target.groupId, id, link));
+                    if (retry.status !== 200) {
+                        console.error(`  ${id}: HTTP ${retry.status} ${retry.message ?? ''}`);
+                        failed++;
+                    }
+                }
+                else {
+                    console.error(`  ${id}: HTTP ${del.status} ${del.message ?? ''}`);
+                    failed++;
+                }
+            }
+            else if (res.status !== 200) {
+                console.error(`  ${id}: HTTP ${res.status} ${res.message ?? ''}`);
+                failed++;
+            }
+        }
+        else if (JSON.stringify(prev.aspects ?? {}) !== JSON.stringify(link.aspects ?? {})) {
+            const upd = await withLinkRateLimitRetry(() => catalog.updateEntryLink(link, Object.keys(link.aspects ?? {})));
+            if (upd.status !== 200) {
+                console.error(`  ${id}: HTTP ${upd.status} ${upd.message ?? ''}`);
+                failed++;
+            }
+        }
+    }
+    return failed;
+}
+class CatalogSync {
+    _catalog;
+    _snapshot;
+    constructor(catalog, snapshot) {
+        this._catalog = catalog;
+        this._snapshot = snapshot;
+    }
+    // Lists metadata in the Catalog service to create or update the local snapshot.
+    async pull(options) {
+        if (this._snapshot.layout instanceof okf_1.OkfLayout) {
+            try {
+                return await this._pullOkf(this._snapshot.layout, options ?? {});
+            }
+            catch (e) {
+                return { success: false, details: e.message };
+            }
+        }
+        try {
+            const listedTypes = this._snapshot.manifest.snapshotConfig?.entries ?? [];
+            const unreadableTypes = new Set();
+            const entries = [];
+            // Loads the type of every entry to pull before fetching any entry, so each
+            // entry is fetched with the same aspect types whatever the listing order.
+            for await (const entry of this._snapshot.manifest.source.entries(this._catalog.context)) {
+                const typeRef = gcp._nameToTypeRef(entry.entryType);
+                // Skips the entry Dataplex keeps for the entry group itself.
+                if (entry.entryType.endsWith('/entryTypes/entrygroup') || unreadableTypes.has(typeRef) ||
+                    (listedTypes.length && !listedTypes.includes(typeRef))) {
+                    continue;
+                }
+                try {
+                    await this._snapshot._loadEntryType(typeRef);
+                }
+                catch (e) {
+                    unreadableTypes.add(typeRef);
+                    console.warn(`Warning: ${e.message}. Entries of this type are not pulled.`);
+                    continue;
+                }
+                entries.push(entry);
+            }
+            // Pull fetches the aspect types it knows: those catalog.yaml lists and
+            // each entry's type requires. It warns about each other aspect type an
+            // entry carries, apart from the scope's system aspects.
+            const aspectTypes = this._snapshot.aspectTypes;
+            const requested = [...aspectTypes.keys()];
+            const system = new Set(this._snapshot.manifest.source.systemAspects);
+            const leftOut = new Map();
+            let wouldCreate = 0;
+            let wouldUpdate = 0;
+            for (const entry of entries) {
+                // TODO: Handle local modification conflicts.
+                // TODO: Handle config changes or service deletions that require removing local entries.
+                const nameParts = entry.name.split('/');
+                const res = await this._catalog.lookupEntry(nameParts[1], nameParts[3], entry.name, { aspects: requested });
+                // The server will respond with 403 permission denied for both resource not exist or 
+                // insufficient permission. We cannot tell if a resource not exist or user does not 
+                // have the access. Thus using 200 for an ensured result.
+                if (res.status != 200 || !res.result) {
+                    continue;
+                }
+                // lookupEntry sets every returned aspect's type.
+                const stored = new Set(Object.values(res.result.aspects ?? {})
+                    .map(a => gcp._nameToTypeRef(a.aspectType)));
+                if (options?.dryRun) {
+                    // Reconstruct and report only; never touch the local snapshot.
+                    const { localName, exists } = this._snapshot._previewStoreEntry(res.result);
+                    console.log(`would ${exists ? 'update' : 'create'} ${localName}`);
+                    if (exists)
+                        wouldUpdate++;
+                    else
+                        wouldCreate++;
+                }
+                else {
+                    await this._snapshot._storeEntry(res.result);
+                }
+                const onEntry = await this._catalog.lookupEntryAspectTypes(nameParts[1], nameParts[3], entry.name);
+                for (const type of onEntry.result ?? []) {
+                    if (!stored.has(type) && !aspectTypes.has(type) && !system.has(type)) {
+                        leftOut.set(type, (leftOut.get(type) ?? 0) + 1);
+                    }
+                }
+            }
+            for (const [type, count] of leftOut) {
+                console.warn(`Warning: pull left out aspect type '${type}' on ${count} ` +
+                    `${count === 1 ? 'entry' : 'entries'}; list it under ` +
+                    `snapshot.aspects in catalog.yaml to pull it.`);
+            }
+            if (options?.dryRun) {
+                console.log(`Dry run: would write ${wouldCreate} new and ${wouldUpdate} updated ` +
+                    `entr${wouldCreate + wouldUpdate === 1 ? 'y' : 'ies'}; no files written (--dry-run).`);
+            }
+            return { success: true };
+        }
+        catch (e) {
+            return { success: false, details: e.message };
+        }
+    }
+    // Pushes local metadata to the Catalog service to publish/deploy it. Under
+    // `validateOnly` the entries are still walked and the pre-flight lookups run,
+    // but no create or update is issued -- mirroring the dry-run contract the
+    // semantic-model legs honor, so the flag behaves the same on every scope.
+    async push(options) {
+        if (this._snapshot.layout instanceof okf_1.OkfLayout) {
+            try {
+                return await this._pushOkf(this._snapshot.layout, options ?? {});
+            }
+            catch (e) {
+                return { success: false, details: e.message };
+            }
+        }
+        const validateOnly = !!options?.validateOnly;
+        const entries = await this._snapshot.listEntries();
+        for (const name of entries) {
+            const entry = await this._snapshot._fetchEntry(name);
+            if (!entry) {
+                // If this was filtered out based on publishing config
+                continue;
+            }
+            // TODO: Track what has changed and do minimal update.
+            // TODO: Handle creates and deletes, as well as partial updates.
+            // TODO: Handle conflicts.
+            const nameParts = entry.name.split('/');
+            const project = nameParts[1];
+            const location = nameParts[3];
+            const exist = await this._catalog.lookupEntry(project, location, entry.name);
+            if (exist.status != 200 || !exist.result) {
+                if (validateOnly) {
+                    // Would create; write nothing under --validate-only.
+                    continue;
+                }
+                const entryGroup = nameParts[5];
+                const entryId = nameParts.slice(7).join('/');
+                const createEntryRes = await this._catalog.createEntry(project, location, entryGroup, entryId, entry);
+                if (createEntryRes.status != 200 || !createEntryRes.result) {
+                    return { success: false, details: `Failed to create entry ${entry.name}: ${createEntryRes.message || createEntryRes.status}` };
+                }
+                continue;
+            }
+            const updateMask = [];
+            const aspectKeys = Object.keys(entry.aspects || {});
+            if (aspectKeys.length) {
+                updateMask.push('aspects');
+            }
+            if (!this._snapshot.manifest.source.ingestedEntries) {
+                if (entry.entrySource) {
+                    updateMask.push('entry_source');
+                }
+                if (entry.parentEntry) {
+                    updateMask.push('parent_entry');
+                }
+            }
+            if (!updateMask.length) {
+                continue;
+            }
+            if (validateOnly) {
+                // Would update; write nothing under --validate-only.
+                continue;
+            }
+            const res = await this._catalog.modifyEntry(project, location, entry, updateMask, aspectKeys);
+            if (res.status !== 200 || !res.result) {
+                return { success: false, details: `Failed to update entry ${name}: ${res.message || res.status}` };
+            }
+        }
+        return { success: true };
+    }
+    async _pullOkf(layout, options) {
+        const { project, location, entryGroup: groupId, } = (0, source_1.entryGroupCoordinates)(this._snapshot.manifest.source);
+        const stubs = [];
+        for await (const stub of this._snapshot.manifest.source.entries(this._catalog.context)) {
+            if (stub.entryType?.endsWith('/entrygroup'))
+                continue;
+            stubs.push(stub);
+        }
+        const fetched = await mapConcurrent(stubs, READ_CONCURRENCY, async (stub) => {
+            const parts = stub.name.split('/');
+            const res = await this._catalog.lookupEntry(parts[1], parts[3], stub.name, { view: 'ALL' });
+            if (res.status !== 200 || !res.result) {
+                throw new Error(`failed to read entry '${stub.name}' (status ${res.status})`);
+            }
+            return res.result;
+        });
+        const planned = [];
+        const pulledNames = new Set();
+        for (const entry of fetched) {
+            if (entry.entryType?.endsWith('/entrygroup'))
+                continue;
+            const localName = this._snapshot.manifest.source.localName(entry);
+            const local = (0, snapshot_1.toLocalEntry)(entry, localName);
+            if (entry.parentEntry) {
+                local.resource.parent =
+                    this._snapshot.manifest.source.localName({ ...entry, name: entry.parentEntry });
+            }
+            // Pre-validate path containment and residue before writing or deleting files.
+            layout.previewConcept(localName, local);
+            planned.push({ entry, localName });
+            pulledNames.add(localName);
+        }
+        const links = await fetchEntryLinks(this._catalog, { project, location, groupId }, planned.map(p => p.entry));
+        const stale = layout.listEntries().filter(name => !pulledNames.has(name));
+        if (options.dryRun) {
+            for (const { localName } of planned) {
+                console.log(`would write ${layout.relativeBundlePath(localName)}`);
+            }
+            if (links.length > 0) {
+                console.log(`would write ${okf_1.BUNDLE_DIR}/${links_1.LINKS_FILE}`);
+            }
+            for (const name of stale) {
+                if (options.forceRemove) {
+                    console.log(`would remove ${layout.relativeBundlePath(name)}`);
+                }
+                else {
+                    console.warn(`Warning: local file '${layout.relativeBundlePath(name)}' is not in ` +
+                        `Knowledge Catalog; re-run with --force-remove to delete it.`);
+                }
+            }
+            console.log(`${planned.length} concept(s); no files written (--dry-run).`);
+            return { success: true };
+        }
+        for (const { entry } of planned) {
+            await this._snapshot._storeEntry(entry);
+        }
+        for (const name of stale) {
+            if (options.forceRemove) {
+                await layout.deleteEntry(name);
+                console.log(`Removed stale concept ${layout.relativeBundlePath(name)}.`);
+            }
+            else {
+                console.warn(`Warning: local file '${layout.relativeBundlePath(name)}' is not in ` +
+                    `Knowledge Catalog; re-run with --force-remove to delete it.`);
+            }
+        }
+        await layout.writeLinks(links);
+        console.log(`Wrote ${planned.length} concept(s) to ${okf_1.BUNDLE_DIR}/.`);
+        return { success: true };
+    }
+    async _pushOkf(layout, options) {
+        const { project, location, entryGroup: groupId, container: entryGroup, } = (0, source_1.entryGroupCoordinates)(this._snapshot.manifest.source);
+        if (!layout.bundleExists()) {
+            return { success: false, details: `no ${okf_1.BUNDLE_DIR}/ directory to push` };
+        }
+        const names = await this._snapshot.listEntries();
+        const entries = [];
+        for (const name of names) {
+            const entry = await this._snapshot._fetchEntry(name);
+            if (!entry)
+                continue;
+            if (entry.entryType.endsWith('/entrygroup'))
+                continue;
+            entries.push(entry);
+        }
+        const { hasLinksConfig, links } = layout.readLinks();
+        const existing = new Map();
+        const remoteEntries = [];
+        for await (const stub of this._snapshot.manifest.source.entries(this._catalog.context)) {
+            existing.set(stub.name, stub);
+            if (stub.entryType?.endsWith('/entrygroup'))
+                continue;
+            if (entryId(stub) === `${groupId}_entry`)
+                continue;
+            remoteEntries.push(stub);
+        }
+        resolveMissingParents(entries, entryGroup, existing);
+        const localNames = new Set(entries.map(e => e.name));
+        const remoteByName = new Map(remoteEntries.map(e => [e.name, e]));
+        const toDelete = remoteEntries
+            .filter(e => !localNames.has(e.name))
+            .sort((a, b) => {
+            const aIsAgents = entryId(a) === 'agents' ? 1 : 0;
+            const bIsAgents = entryId(b) === 'agents' ? 1 : 0;
+            if (aIsAgents !== bIsAgents)
+                return aIsAgents - bIsAgents;
+            const depthDiff = entryDepth(b, remoteByName) - entryDepth(a, remoteByName);
+            if (depthDiff !== 0)
+                return depthDiff;
+            const slashDiff = entryId(b).split('/').length - entryId(a).split('/').length;
+            return slashDiff !== 0 ? slashDiff : entryId(a).localeCompare(entryId(b));
+        });
+        if (!entries.length && !toDelete.length && !links.length) {
+            return {
+                success: false,
+                details: `no OKF concepts found in ${okf_1.BUNDLE_DIR}/`,
+            };
+        }
+        const upsertChanges = entries.map(entry => toChange(entry, entryGroup, existing));
+        const deleteChanges = toDelete.map(e => ({
+            deleteEntryRequest: { name: e.name },
+        }));
+        const changes = [...upsertChanges, ...deleteChanges];
+        const creates = upsertChanges.filter(c => c.createEntryRequest).length;
+        const updates = upsertChanges.length - creates;
+        const deletes = deleteChanges.length;
+        if (options.validateOnly) {
+            for (const entry of entries)
+                console.log(`would write ${entry.name}`);
+            for (const entry of toDelete)
+                console.log(`would delete ${entry.name}`);
+            for (const link of links)
+                console.log(`would link ${link.name}`);
+            console.log(`${changes.length} change(s): ${creates} create, ${updates} update, ` +
+                `${deletes} delete. Nothing sent (dry run).`);
+            return { success: true };
+        }
+        let failed = 0;
+        for (let i = 0; i < changes.length; i += BATCH_MAX_CHANGES) {
+            failed += await applyChunk(this._catalog, { project, location, groupId }, changes.slice(i, i + BATCH_MAX_CHANGES));
+        }
+        if (hasLinksConfig) {
+            const deletedNames = new Set(toDelete.map(e => e.name));
+            const survivingExisting = new Map([...existing.entries()].filter(([k]) => !deletedNames.has(k)));
+            failed += await reconcileLinks(this._catalog, { project, location, groupId }, links, entries, survivingExisting);
+        }
+        console.log(`Pushed ${changes.length - Math.min(failed, changes.length)}/${changes.length} change(s) to ` +
+            `${groupId} (${creates} create, ${updates} update, ${deletes} delete).`);
+        return failed ? { success: false } : { success: true };
+    }
+    async validate() {
+        throw new Error('Not yet implemented');
+    }
+    async status() {
+        throw new Error('Not yet implemented');
+    }
+}
+exports.CatalogSync = CatalogSync;