@kweaver-ai/kweaver-sdk 0.8.3 → 0.8.4

package/dist/commands/ds.js

@@ -1,443 +1,19 @@
-import { createInterface } from "node:readline";
+// Internal CSV-import helpers — formerly the `kweaver ds` command. The user-facing
+// `ds` command group has been removed; these helpers are now consumed by
+// `kweaver bkn create-from-csv` and by the import-csv tests.
+//
+// The flow still talks to the legacy `/datasources` backend endpoints. SDK-level
+// shims for those endpoints remain available for users on older backends; see
+// `api/datasources.ts` and `resources/datasources.ts` (both @deprecated).
 import { statSync } from "node:fs";
 import { glob } from "node:fs/promises";
 import { resolve as resolvePath } from "node:path";
-import { ensureValidToken, formatHttpError, with401RefreshRetry } from "../auth/oauth.js";
-import { testDatasource, createDatasource, listDatasources, getDatasource, deleteDatasource, } from "../api/datasources.js";
-import { listTablesWithColumns, scanMetadata } from "../api/vega.js";
-import { formatCallOutput } from "./call.js";
+import { ensureValidToken, formatHttpError } from "../auth/oauth.js";
+import { getDatasource } from "../api/datasources.js";
+import { scanMetadata } from "../api/vega.js";
 import { resolveBusinessDomain } from "../config/store.js";
-import { assertVegaCatalogId } from "./bkn-utils.js";
 import { parseCsvFile, buildTableName, splitBatches, buildFieldMappings, buildDagBody, } from "./import-csv.js";
 import { executeDataflow } from "../api/dataflow.js";
-import { renderHelp } from "../help/format.js";
-const DS_HELP = renderHelp({
-    tagline: "Manage datasources — list, get, delete, tables, connect, import-csv",
-    usage: "kweaver ds <subcommand> [flags]",
-    sections: [
-        {
-            title: "AVAILABLE COMMANDS",
-            items: [
-                { name: "list", desc: "List datasources" },
-                { name: "get", desc: "Get datasource details" },
-                { name: "delete", desc: "Delete a datasource" },
-                { name: "tables", desc: "List tables with columns" },
-                { name: "connect", desc: "Test, register and discover; reuses by (type,host,port,db,account) unless --force-new" },
-                { name: "import-csv", desc: "Import CSV files into datasource tables via dataflow API" },
-            ],
-        },
-    ],
-    inheritedFlags: "--base-url, --token, --user, --help",
-    examples: [
-        "kweaver ds list --keyword mysql",
-        "kweaver ds connect mysql 127.0.0.1 3306 mydb --account root --password ******",
-        "kweaver ds tables <id> --keyword users",
-    ],
-    learnMore: ["Use `kweaver ds <subcommand> --help` for flag details"],
-});
-function confirmYes(prompt) {
-    return new Promise((resolve) => {
-        const rl = createInterface({ input: process.stdin, output: process.stdout });
-        rl.question(`${prompt} [y/N] `, (answer) => {
-            rl.close();
-            const trimmed = answer.trim().toLowerCase();
-            resolve(trimmed === "y" || trimmed === "yes");
-        });
-    });
-}
-function extractDatasourceId(body) {
-    const parsed = JSON.parse(body);
-    const item = Array.isArray(parsed) ? parsed[0] : parsed;
-    if (!item || typeof item !== "object")
-        return "";
-    const id = item.id ?? item.ds_id;
-    return id != null ? String(id) : "";
-}
-export async function runDsCommand(args) {
-    const [subcommand, ...rest] = args;
-    if (!subcommand || subcommand === "--help" || subcommand === "-h") {
-        console.log(DS_HELP);
-        return 0;
-    }
-    const dispatch = () => {
-        if (subcommand === "list")
-            return runDsListCommand(rest);
-        if (subcommand === "get")
-            return runDsGetCommand(rest);
-        if (subcommand === "delete")
-            return runDsDeleteCommand(rest);
-        if (subcommand === "tables")
-            return runDsTablesCommand(rest);
-        if (subcommand === "connect")
-            return runDsConnectCommand(rest);
-        if (subcommand === "import-csv")
-            return runDsImportCsvCommand(rest);
-        return Promise.resolve(-1);
-    };
-    try {
-        return await with401RefreshRetry(async () => {
-            const code = await dispatch();
-            if (code === -1) {
-                console.error(`Unknown ds subcommand: ${subcommand}`);
-                return 1;
-            }
-            return code;
-        });
-    }
-    catch (error) {
-        console.error(formatHttpError(error));
-        return 1;
-    }
-}
-export function parseDsListArgs(args) {
-    let keyword;
-    let type;
-    let businessDomain = "";
-    let pretty = true;
-    for (let i = 0; i < args.length; i += 1) {
-        const arg = args[i];
-        if (arg === "--help" || arg === "-h")
-            throw new Error("help");
-        if (arg === "--keyword" && args[i + 1]) {
-            keyword = args[++i];
-            continue;
-        }
-        if (arg === "--type" && args[i + 1]) {
-            type = args[++i];
-            continue;
-        }
-        if ((arg === "-bd" || arg === "--biz-domain") && args[i + 1]) {
-            businessDomain = args[++i];
-            continue;
-        }
-        if (arg === "--pretty") {
-            pretty = true;
-            continue;
-        }
-    }
-    if (!businessDomain)
-        businessDomain = resolveBusinessDomain();
-    return { keyword, type, businessDomain, pretty };
-}
-async function runDsListCommand(args) {
-    try {
-        const opts = parseDsListArgs(args);
-        const token = await ensureValidToken();
-        const body = await listDatasources({
-            baseUrl: token.baseUrl,
-            accessToken: token.accessToken,
-            keyword: opts.keyword,
-            type: opts.type,
-            businessDomain: opts.businessDomain,
-        });
-        console.log(formatCallOutput(body, opts.pretty));
-        return 0;
-    }
-    catch (error) {
-        if (error instanceof Error && error.message === "help") {
-            console.log(renderHelp({
-                tagline: "List datasources",
-                usage: "kweaver ds list [options]",
-                flags: [
-                    { name: "--keyword <s>", desc: "Filter by keyword" },
-                    { name: "--type <s>", desc: "Filter by database type" },
-                    { name: "-bd, --biz-domain", desc: "Business domain (default: bd_public)" },
-                    { name: "--pretty", desc: "Pretty-print JSON (default)" },
-                ],
-                inheritedFlags: "--base-url, --token, --user, --help",
-            }));
-            return 0;
-        }
-        throw error;
-    }
-}
-async function runDsGetCommand(args) {
-    const id = args.find((a) => !a.startsWith("-"));
-    if (!id) {
-        console.error("Usage: kweaver ds get <id>");
-        return 1;
-    }
-    const token = await ensureValidToken();
-    const body = await getDatasource({
-        baseUrl: token.baseUrl,
-        accessToken: token.accessToken,
-        id,
-    });
-    console.log(formatCallOutput(body, true));
-    return 0;
-}
-async function runDsDeleteCommand(args) {
-    let id = "";
-    let yes = false;
-    for (let i = 0; i < args.length; i += 1) {
-        const arg = args[i];
-        if (arg === "--yes" || arg === "-y")
-            yes = true;
-        else if (!arg.startsWith("-"))
-            id = arg;
-    }
-    if (!id) {
-        console.error("Usage: kweaver ds delete <id> [-y]");
-        return 1;
-    }
-    if (!yes) {
-        const confirmed = await confirmYes("Are you sure you want to delete this datasource?");
-        if (!confirmed) {
-            console.error("Aborted.");
-            return 1;
-        }
-    }
-    const token = await ensureValidToken();
-    await deleteDatasource({
-        baseUrl: token.baseUrl,
-        accessToken: token.accessToken,
-        id,
-    });
-    console.error(`Deleted ${id}`);
-    return 0;
-}
-async function runDsTablesCommand(args) {
-    let id = "";
-    let keyword;
-    let pretty = true;
-    for (let i = 0; i < args.length; i += 1) {
-        const arg = args[i];
-        if (arg === "--keyword" && args[i + 1]) {
-            keyword = args[++i];
-            continue;
-        }
-        if (arg === "--pretty") {
-            pretty = true;
-            continue;
-        }
-        if (!arg.startsWith("-"))
-            id = arg;
-    }
-    if (!id) {
-        console.error("Usage: kweaver ds tables <vega-catalog-id> [--keyword X]\n" +
-            "  <vega-catalog-id> is a vega catalog id (use `kweaver vega catalog list` to find one;\n" +
-            "  legacy data-connection datasource UUIDs are no longer accepted).");
-        return 1;
-    }
-    assertVegaCatalogId(id);
-    const token = await ensureValidToken();
-    const body = await listTablesWithColumns({
-        baseUrl: token.baseUrl,
-        accessToken: token.accessToken,
-        id,
-        keyword,
-    });
-    console.log(formatCallOutput(body, pretty));
-    return 0;
-}
-export function findExistingDatasource(listBody, sig) {
-    const parsed = JSON.parse(listBody);
-    const entries = Array.isArray(parsed) ? parsed : (parsed.entries ?? []);
-    const tupleMatch = entries.find((e) => e.id &&
-        e.type === sig.type &&
-        e.bin_data?.host === sig.host &&
-        Number(e.bin_data?.port) === Number(sig.port) &&
-        e.bin_data?.database_name === sig.database &&
-        e.bin_data?.account === sig.account);
-    if (tupleMatch) {
-        return {
-            id: String(tupleMatch.id),
-            name: String(tupleMatch.name ?? ""),
-            matchedByName: tupleMatch.name === sig.name,
-            matchedByTuple: true,
-        };
-    }
-    if (sig.name) {
-        const nameMatch = entries.find((e) => e.id && e.name === sig.name);
-        if (nameMatch) {
-            return {
-                id: String(nameMatch.id),
-                name: String(nameMatch.name),
-                matchedByName: true,
-                matchedByTuple: false,
-            };
-        }
-    }
-    return undefined;
-}
-export function findDatasourceIdByName(listBody, name) {
-    const parsed = JSON.parse(listBody);
-    const entries = Array.isArray(parsed) ? parsed : (parsed.entries ?? []);
-    const hit = entries.find((e) => e.id && e.name === name);
-    return hit?.id ? String(hit.id) : undefined;
-}
-function isDuplicateNameError(err) {
-    if (!err || typeof err !== "object")
-        return false;
-    // HttpError.message is just "HTTP 400 ..."; the description lives in body.
-    const status = "status" in err ? Number(err.status) : NaN;
-    const body = "body" in err ? String(err.body) : "";
-    if (status !== 400)
-        return false;
-    return /数据源名称已存在|datasource name.*exist|already exists/i.test(body);
-}
-async function runDsConnectCommand(args) {
-    let dbType = "";
-    let host = "";
-    let port = 0;
-    let database = "";
-    let account = "";
-    let password = "";
-    let schema;
-    let name;
-    let forceNew = false;
-    for (let i = 0; i < args.length; i += 1) {
-        const arg = args[i];
-        if (arg === "--account" && args[i + 1]) {
-            account = args[++i];
-            continue;
-        }
-        if (arg === "--password" && args[i + 1]) {
-            password = args[++i];
-            continue;
-        }
-        if (arg === "--schema" && args[i + 1]) {
-            schema = args[++i];
-            continue;
-        }
-        if (arg === "--name" && args[i + 1]) {
-            name = args[++i];
-            continue;
-        }
-        if (arg === "--force-new") {
-            forceNew = true;
-            continue;
-        }
-        if (arg === "--reuse-existing") {
-            forceNew = false;
-            continue;
-        }
-        if (!arg.startsWith("-")) {
-            if (!dbType)
-                dbType = arg;
-            else if (!host)
-                host = arg;
-            else if (port === 0)
-                port = parseInt(arg, 10);
-            else if (!database)
-                database = arg;
-        }
-    }
-    if (!dbType || !host || !database || !account || !password) {
-        console.error("Usage: kweaver ds connect <db_type> <host> <port> <database> --account X --password Y [--schema Z] [--name N] [--reuse-existing|--force-new]");
-        return 1;
-    }
-    if (Number.isNaN(port) || port < 1) {
-        console.error("Invalid port");
-        return 1;
-    }
-    const token = await ensureValidToken();
-    const base = { baseUrl: token.baseUrl, accessToken: token.accessToken };
-    const dsName = name ?? database;
-    // Pre-flight dedup: connection-tuple match is the silent-orphan vector.
-    // Backend already rejects duplicate names with 400, but won't notice
-    // tuple collisions, so we own that check.
-    if (!forceNew) {
-        const listBody = await listDatasources({ ...base });
-        const hit = findExistingDatasource(listBody, {
-            type: dbType,
-            host,
-            port,
-            database,
-            account,
-            name: dsName,
-        });
-        if (hit) {
-            const why = hit.matchedByTuple
-                ? "matched by (type,host,port,database,account)"
-                : "matched by --name";
-            console.error(`Reusing existing datasource ${hit.id} (${hit.name}); ${why}. Use --force-new to override.`);
-            return printDsConnectOutput(base, hit.id);
-        }
-    }
-    console.error("Testing connectivity ...");
-    await testDatasource({
-        ...base,
-        type: dbType,
-        host,
-        port,
-        database,
-        account,
-        password,
-        schema,
-    });
-    let dsId = "";
-    try {
-        const createBody = await createDatasource({
-            ...base,
-            name: dsName,
-            type: dbType,
-            host,
-            port,
-            database,
-            account,
-            password,
-            schema,
-        });
-        dsId = extractDatasourceId(createBody);
-    }
-    catch (err) {
-        // Backend checks name uniqueness but not tuple. If we raced another caller
-        // (or tuple match got disabled by --force-new and the name still collides),
-        // turn the raw 400 into a useful pointer to the existing id.
-        if (isDuplicateNameError(err)) {
-            // Backend rejected the name; look it up specifically (not by tuple —
-            // sibling ds sharing the same connection would mislead the pointer).
-            const listBody = await listDatasources({ ...base });
-            const existingId = findDatasourceIdByName(listBody, dsName);
-            if (existingId) {
-                console.error(`Datasource name '${dsName}' already exists as ${existingId}. Re-run without --force-new to reuse it, or pick a different --name.`);
-                return 1;
-            }
-        }
-        throw err;
-    }
-    if (!dsId) {
-        console.error("Failed to get datasource ID from create response");
-        return 1;
-    }
-    return printDsConnectOutput(base, dsId);
-}
-async function printDsConnectOutput(base, dsId) {
-    const tablesBody = await listTablesWithColumns({ ...base, id: dsId });
-    const tables = JSON.parse(tablesBody);
-    const output = {
-        datasource_id: dsId,
-        tables: tables.map((t) => ({
-            name: t.name,
-            ...(t.primaryKeys && t.primaryKeys.length > 0 ? { primary_keys: t.primaryKeys } : {}),
-            columns: t.columns.map((c) => ({
-                name: c.name,
-                type: c.type,
-                comment: c.comment,
-                ...(c.isPrimaryKey ? { is_primary_key: true } : {}),
-            })),
-        })),
-    };
-    console.log(JSON.stringify(output, null, 2));
-    return 0;
-}
-// ── import-csv ────────────────────────────────────────────────────────────────
-const IMPORT_CSV_HELP = renderHelp({
-    tagline: "Import CSV files into datasource tables via dataflow API",
-    usage: "kweaver ds import-csv <ds-id> --files <glob_or_list> [flags]",
-    flags: [
-        { name: "--files <s>", desc: "CSV file paths — comma-separated or glob pattern (required)" },
-        { name: "--table-prefix <s>", desc: "Table name prefix (default: none)" },
-        { name: "--batch-size <n>", desc: "Rows per batch (default: 500, range: 1-10000)" },
-        { name: "-bd, --biz-domain <s>", desc: "Business domain (default: bd_public)" },
-    ],
-    inheritedFlags: "--base-url, --token, --user, --help",
-    examples: [
-        "kweaver ds import-csv ds-123 --files './data/*.csv'",
-        "kweaver ds import-csv ds-123 --files ./a.csv,./b.csv --table-prefix raw_ --batch-size 1000",
-    ],
-});
 export function parseImportCsvArgs(args) {
     let datasourceId = "";
     let files = "";
@@ -508,13 +84,15 @@ export async function runDsImportCsv(args) {
     }
     catch (error) {
         if (error instanceof Error && error.message === "help") {
-            console.log(IMPORT_CSV_HELP);
+            // The user-facing `ds import-csv` CLI is gone; this path is only reachable
+            // if a caller explicitly passes --help into the helper. Print a redirect.
+            console.error("CSV import is now an internal helper. Use `kweaver bkn create-from-csv` instead.");
             return { code: 0, tables: [], failed: [], tableColumns: {}, sampleRows: {} };
         }
         throw error;
     }
     if (!options.datasourceId) {
-        console.error("Usage: kweaver ds import-csv <ds-id> --files <glob_or_list> [options]");
+        console.error("runDsImportCsv: missing datasource/catalog id (positional argument required)");
         return { code: 1, tables: [], failed: [], tableColumns: {}, sampleRows: {} };
     }
     if (!options.files) {
@@ -611,9 +189,10 @@ export async function runDsImportCsv(args) {
     if (failed.length > 0) {
         console.error(`Failed tables: ${failed.join(", ")}`);
     }
-    // Refresh the platform metadata catalog so the freshly imported tables
-    // are visible to ds tables / bkn create-from-ds without manual scan.
-    // Best-effort: scan failures shouldn't mask a successful import.
+    // Refresh the platform metadata catalog so the freshly imported tables are
+    // visible to downstream commands (e.g. `bkn create-from-catalog`) without
+    // requiring a manual scan. Best-effort: scan failures don't mask a
+    // successful import.
     if (succeeded.length > 0) {
         process.stderr.write("Scanning datasource metadata ...\n");
         try {
@@ -630,12 +209,3 @@ export async function runDsImportCsv(args) {
     }
     return { code: failed.length > 0 ? 1 : 0, tables: succeeded, failed, tableColumns, sampleRows };
 }
-export async function runDsImportCsvCommand(args) {
-    const result = await runDsImportCsv(args);
-    console.log(JSON.stringify({
-        tables: result.tables,
-        failed: result.failed,
-        summary: { succeeded: result.tables.length, failed: result.failed.length },
-    }, null, 2));
-    return result.code;
-}

package/dist/commands/explore-bkn.d.ts

@@ -30,6 +30,12 @@ export interface ExploreMeta {
         id: string;
         name: string;
     }>;
+    conceptGroups: Array<{
+        id: string;
+        name: string;
+        color?: string;
+        objectTypeIds: string[];
+    }>;
 }
 export interface ExploreOt {
     id: string;
@@ -63,7 +69,7 @@ export interface ExploreStats {
 }
 export declare const EXPLORE_BOOTSTRAP_RETRY_DELAY_MS = 300;
 export declare const EXPLORE_BOOTSTRAP_MAX_ATTEMPTS = 2;
-export declare function buildMeta(knRaw: string, otRaw: string, rtRaw: string, atRaw: string): ExploreMeta;
+export declare function buildMeta(knRaw: string, otRaw: string, rtRaw: string, atRaw: string, cgRaw?: string): ExploreMeta;
 export declare function isRetryableExploreBootstrapError(error: unknown): boolean;
 export declare function loadExploreMetaWithRetry(token: {
     baseUrl: string;

package/dist/commands/explore-bkn.js

@@ -2,11 +2,12 @@ import { HttpError } from "../utils/http.js";
 import { getKnowledgeNetwork, listObjectTypes, listRelationTypes, listActionTypes, } from "../api/knowledge-networks.js";
 import { objectTypeQuery, objectTypeProperties, subgraph } from "../api/ontology-query.js";
 import { semanticSearch } from "../api/semantic-search.js";
+import { listConceptGroups } from "../api/bkn-backend.js";
 // ── Constants ───────────────────────────────────────────────────────────────
 export const EXPLORE_BOOTSTRAP_RETRY_DELAY_MS = 300;
 export const EXPLORE_BOOTSTRAP_MAX_ATTEMPTS = 2;
 // ── Meta builder ────────────────────────────────────────────────────────────
-export function buildMeta(knRaw, otRaw, rtRaw, atRaw) {
+export function buildMeta(knRaw, otRaw, rtRaw, atRaw, cgRaw) {
     const kn = JSON.parse(knRaw);
     const otParsed = JSON.parse(otRaw);
     const otItems = (Array.isArray(otParsed) ? otParsed
@@ -54,8 +55,29 @@ export function buildMeta(knRaw, otRaw, rtRaw, atRaw) {
             id: a.id,
             name: a.name,
         })),
+        conceptGroups: parseConceptGroups(cgRaw),
     };
 }
+function parseConceptGroups(cgRaw) {
+    if (!cgRaw)
+        return [];
+    try {
+        const parsed = JSON.parse(cgRaw);
+        const items = (Array.isArray(parsed) ? parsed
+            : Array.isArray(parsed.entries) ? parsed.entries
+                : Array.isArray(parsed.concept_groups) ? parsed.concept_groups
+                    : []);
+        return items.map((g) => ({
+            id: g.id,
+            name: g.name,
+            ...(g.color ? { color: g.color } : {}),
+            objectTypeIds: Array.isArray(g.object_type_ids) ? g.object_type_ids : [],
+        }));
+    }
+    catch {
+        return [];
+    }
+}
 // ── Bootstrap helpers ───────────────────────────────────────────────────────
 function getErrorMessage(error) {
     const parts = [];
@@ -100,7 +122,7 @@ function sleep(ms) {
 export async function loadExploreMetaWithRetry(token, knId, businessDomain) {
     for (let attempt = 1; attempt <= EXPLORE_BOOTSTRAP_MAX_ATTEMPTS; attempt++) {
         try {
-            const [knRaw, otRaw, rtRaw, atRaw] = await Promise.all([
+            const [knRaw, otRaw, rtRaw, atRaw, cgRaw] = await Promise.all([
                 getKnowledgeNetwork({
                     baseUrl: token.baseUrl,
                     accessToken: token.accessToken,
@@ -126,8 +148,15 @@ export async function loadExploreMetaWithRetry(token, knId, businessDomain) {
                     knId,
                     businessDomain,
                 }),
+                // Concept groups are optional — don't fail the whole load if missing
+                listConceptGroups({
+                    baseUrl: token.baseUrl,
+                    accessToken: token.accessToken,
+                    knId,
+                    businessDomain,
+                }).catch(() => ""),
             ]);
-            return buildMeta(knRaw, otRaw, rtRaw, atRaw);
+            return buildMeta(knRaw, otRaw, rtRaw, atRaw, cgRaw);
         }
         catch (error) {
             if (attempt >= EXPLORE_BOOTSTRAP_MAX_ATTEMPTS || !isRetryableExploreBootstrapError(error)) {

package/dist/resources/datasources.d.ts

@@ -1,4 +1,11 @@
 import type { ClientContext } from "../client.js";
+/**
+ * @deprecated Datasource is a legacy concept. The platform has migrated
+ * data-connection management to vega catalogs. Use `client.vega.catalogs`
+ * for catalog CRUD and `client.resources` for resource queries instead.
+ * This class is kept as a thin back-compat layer and may be removed in a
+ * future major release.
+ */
 export declare class DataSourcesResource {
     private readonly ctx;
     constructor(ctx: ClientContext);

package/dist/resources/datasources.js

@@ -1,5 +1,12 @@
 import { testDatasource, createDatasource, listDatasources, getDatasource, deleteDatasource, listTables, } from "../api/datasources.js";
 import { listTablesWithColumns, scanMetadata } from "../api/vega.js";
+/**
+ * @deprecated Datasource is a legacy concept. The platform has migrated
+ * data-connection management to vega catalogs. Use `client.vega.catalogs`
+ * for catalog CRUD and `client.resources` for resource queries instead.
+ * This class is kept as a thin back-compat layer and may be removed in a
+ * future major release.
+ */
 export class DataSourcesResource {
     ctx;
     constructor(ctx) {