mdkg 0.1.2 → 0.1.4

package/dist/core/config.js

@@ -22,7 +22,12 @@ const PACK_EDGE_KEYS = new Set([
 const NEXT_WORK_STRATEGIES = new Set(["chain_then_priority"]);
 const WORKSPACE_VISIBILITY_VALUES = new Set(["private", "internal", "public"]);
 const BUNDLE_PROFILE_VALUES = new Set(["private", "public"]);
+const INDEX_BACKEND_VALUES = new Set(["json", "sqlite"]);
+const SUBGRAPH_PERMISSION_VALUES = new Set(["read", "request", "propose", "mutate"]);
 const DEFAULT_ARCHIVE_LARGE_CACHE_WARNING_BYTES = 26214400;
+const DEFAULT_SQLITE_COMMIT_WARNING_BYTES = 52428800;
+const DEFAULT_LOCK_TIMEOUT_MS = 10000;
+const DEFAULT_SUBGRAPH_MAX_STALE_SECONDS = 3600;
 function isObject(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -169,16 +174,16 @@ function validateWorkspaceAlias(alias, errors) {
         errors.push(`workspaces.${alias} alias must be lowercase and use [a-z0-9_]`);
     }
 }
-function validateBundleImportAlias(alias, workspaces, errors) {
+function validateSubgraphAlias(alias, workspaces, errors) {
     if (alias === "all") {
-        errors.push("bundle_imports.all alias is reserved");
+        errors.push("subgraphs.all alias is reserved");
         return;
     }
     if (alias !== alias.toLowerCase() || !WORKSPACE_ALIAS_RE.test(alias)) {
-        errors.push(`bundle_imports.${alias} alias must be lowercase and use [a-z0-9_]`);
+        errors.push(`subgraphs.${alias} alias must be lowercase and use [a-z0-9_]`);
     }
     if (workspaces[alias]) {
-        errors.push(`bundle_imports.${alias} must not collide with workspaces.${alias}`);
+        errors.push(`subgraphs.${alias} must not collide with workspaces.${alias}`);
     }
 }
 function requireContainedPath(value, path, errors) {
@@ -213,9 +218,15 @@ function validateConfigSchema(raw) {
     const bundlesRaw = raw.bundles === undefined
         ? { output_dir: ".mdkg/bundles", default_profile: "private" }
         : requireObject(raw.bundles, "bundles", errors);
-    const bundleImportsRaw = raw.bundle_imports === undefined
-        ? {}
-        : requireObject(raw.bundle_imports, "bundle_imports", errors);
+    if (raw.bundle_imports !== undefined && raw.subgraphs !== undefined) {
+        errors.push("config must not define both subgraphs and legacy bundle_imports");
+    }
+    const subgraphsFromLegacyBundleImports = raw.subgraphs === undefined && raw.bundle_imports !== undefined;
+    const subgraphsRaw = raw.subgraphs === undefined
+        ? raw.bundle_imports === undefined
+            ? {}
+            : requireObject(raw.bundle_imports, "bundle_imports", errors)
+        : requireObject(raw.subgraphs, "subgraphs", errors);
     const packRaw = requireObject(raw.pack, "pack", errors);
     const templatesRaw = requireObject(raw.templates, "templates", errors);
     const workRaw = requireObject(raw.work, "work", errors);
@@ -231,7 +242,19 @@ function validateConfigSchema(raw) {
         ? {
             auto_reindex: requireBoolean(indexRaw.auto_reindex, "index.auto_reindex", errors),
             tolerant: requireBoolean(indexRaw.tolerant, "index.tolerant", errors),
+            backend: indexRaw.backend === undefined
+                ? "json"
+                : requireStringInSet(indexRaw.backend, "index.backend", INDEX_BACKEND_VALUES, errors),
             global_index_path: requireContainedPath(indexRaw.global_index_path, "index.global_index_path", errors),
+            sqlite_path: indexRaw.sqlite_path === undefined
+                ? ".mdkg/index/mdkg.sqlite"
+                : requireContainedPath(indexRaw.sqlite_path, "index.sqlite_path", errors),
+            sqlite_commit_warning_bytes: indexRaw.sqlite_commit_warning_bytes === undefined
+                ? DEFAULT_SQLITE_COMMIT_WARNING_BYTES
+                : requireNonNegativeInteger(indexRaw.sqlite_commit_warning_bytes, "index.sqlite_commit_warning_bytes", errors),
+            lock_timeout_ms: indexRaw.lock_timeout_ms === undefined
+                ? DEFAULT_LOCK_TIMEOUT_MS
+                : requirePositiveInteger(indexRaw.lock_timeout_ms, "index.lock_timeout_ms", errors),
         }
         : undefined;
     const capabilities = capabilitiesRaw
@@ -374,48 +397,96 @@ function validateConfigSchema(raw) {
             errors.push('workspaces.root.mdkg_dir must be ".mdkg"');
         }
     }
-    const bundle_imports = {};
-    if (bundleImportsRaw) {
-        for (const [alias, entry] of Object.entries(bundleImportsRaw)) {
-            validateBundleImportAlias(alias, workspaces, errors);
-            const rawImport = requireObject(entry, `bundle_imports.${alias}`, errors);
-            if (!rawImport) {
+    const subgraphs = {};
+    if (subgraphsRaw) {
+        for (const [alias, entry] of Object.entries(subgraphsRaw)) {
+            validateSubgraphAlias(alias, workspaces, errors);
+            const basePath = subgraphsFromLegacyBundleImports ? `bundle_imports.${alias}` : `subgraphs.${alias}`;
+            const rawSubgraph = requireObject(entry, basePath, errors);
+            if (!rawSubgraph) {
                 continue;
             }
-            const importPath = requireContainedPath(rawImport.path, `bundle_imports.${alias}.path`, errors);
-            const enabled = rawImport.enabled === undefined
+            const enabled = rawSubgraph.enabled === undefined
                 ? true
-                : requireBoolean(rawImport.enabled, `bundle_imports.${alias}.enabled`, errors);
-            const visibility = rawImport.visibility === undefined
+                : requireBoolean(rawSubgraph.enabled, `${basePath}.enabled`, errors);
+            const visibility = rawSubgraph.visibility === undefined
                 ? "private"
-                : requireStringInSet(rawImport.visibility, `bundle_imports.${alias}.visibility`, WORKSPACE_VISIBILITY_VALUES, errors);
-            const expectedProfile = rawImport.expected_profile === undefined
-                ? "private"
-                : requireStringInSet(rawImport.expected_profile, `bundle_imports.${alias}.expected_profile`, BUNDLE_PROFILE_VALUES, errors);
-            const sourcePath = rawImport.source_path === undefined
-                ? undefined
-                : requireContainedPath(rawImport.source_path, `bundle_imports.${alias}.source_path`, errors);
-            const sourceRepo = rawImport.source_repo === undefined
+                : requireStringInSet(rawSubgraph.visibility, `${basePath}.visibility`, WORKSPACE_VISIBILITY_VALUES, errors);
+            const permissions = rawSubgraph.permissions === undefined || subgraphsFromLegacyBundleImports
+                ? ["read"]
+                : requireKnownLowercaseUniqueStringArray(rawSubgraph.permissions, `${basePath}.permissions`, SUBGRAPH_PERMISSION_VALUES, errors);
+            const maxStaleSeconds = rawSubgraph.max_stale_seconds === undefined
+                ? DEFAULT_SUBGRAPH_MAX_STALE_SECONDS
+                : requirePositiveInteger(rawSubgraph.max_stale_seconds, `${basePath}.max_stale_seconds`, errors);
+            const sourcePath = rawSubgraph.source_path === undefined
                 ? undefined
-                : requireString(rawImport.source_repo, `bundle_imports.${alias}.source_repo`, errors);
-            const maxStaleSeconds = rawImport.max_stale_seconds === undefined
+                : requireContainedPath(rawSubgraph.source_path, `${basePath}.source_path`, errors);
+            const sourceRepo = rawSubgraph.source_repo === undefined
                 ? undefined
-                : requirePositiveInteger(rawImport.max_stale_seconds, `bundle_imports.${alias}.max_stale_seconds`, errors);
-            if (visibility !== undefined &&
-                expectedProfile !== undefined &&
-                visibility !== "private" &&
-                expectedProfile !== "public") {
-                errors.push(`bundle_imports.${alias}.expected_profile must be public when visibility is ${visibility}`);
+                : requireString(rawSubgraph.source_repo, `${basePath}.source_repo`, errors);
+            const rawSources = subgraphsFromLegacyBundleImports
+                ? [{ path: rawSubgraph.path, enabled: true, expected_profile: rawSubgraph.expected_profile }]
+                : rawSubgraph.sources;
+            if (!Array.isArray(rawSources)) {
+                errors.push(`${basePath}.sources must be an array`);
+                continue;
+            }
+            if (rawSources.length === 0) {
+                errors.push(`${basePath}.sources must not be empty`);
+                continue;
+            }
+            const sources = [];
+            const labels = new Set();
+            for (let i = 0; i < rawSources.length; i += 1) {
+                const sourceBasePath = `${basePath}.sources[${i}]`;
+                const rawSource = requireObject(rawSources[i], sourceBasePath, errors);
+                if (!rawSource) {
+                    continue;
+                }
+                const sourceBundlePath = requireContainedPath(rawSource.path, `${sourceBasePath}.path`, errors);
+                const sourceEnabled = rawSource.enabled === undefined
+                    ? true
+                    : requireBoolean(rawSource.enabled, `${sourceBasePath}.enabled`, errors);
+                const expectedProfile = rawSource.expected_profile === undefined
+                    ? "private"
+                    : requireStringInSet(rawSource.expected_profile, `${sourceBasePath}.expected_profile`, BUNDLE_PROFILE_VALUES, errors);
+                const label = rawSource.label === undefined
+                    ? undefined
+                    : requireString(rawSource.label, `${sourceBasePath}.label`, errors);
+                if (label !== undefined) {
+                    if (label.trim().length === 0) {
+                        errors.push(`${sourceBasePath}.label must not be empty`);
+                    }
+                    if (labels.has(label)) {
+                        errors.push(`${basePath}.sources label must be unique: ${label}`);
+                    }
+                    labels.add(label);
+                }
+                if (visibility !== undefined &&
+                    expectedProfile !== undefined &&
+                    sourceEnabled !== false &&
+                    visibility !== "private" &&
+                    expectedProfile !== "public") {
+                    errors.push(`${sourceBasePath}.expected_profile must be public when ${basePath}.visibility is ${visibility}`);
+                }
+                if (sourceBundlePath && sourceEnabled !== undefined && expectedProfile) {
+                    sources.push({
+                        path: sourceBundlePath,
+                        enabled: sourceEnabled,
+                        expected_profile: expectedProfile,
+                        ...(label ? { label } : {}),
+                    });
+                }
             }
-            if (importPath && enabled !== undefined && visibility && expectedProfile) {
-                bundle_imports[alias] = {
-                    path: importPath,
+            if (enabled !== undefined && visibility && permissions && maxStaleSeconds !== undefined && sources.length > 0) {
+                subgraphs[alias] = {
                     enabled,
                     visibility: visibility,
-                    expected_profile: expectedProfile,
+                    permissions,
+                    max_stale_seconds: maxStaleSeconds,
                     ...(sourcePath ? { source_path: sourcePath } : {}),
                     ...(sourceRepo ? { source_repo: sourceRepo } : {}),
-                    ...(maxStaleSeconds !== undefined ? { max_stale_seconds: maxStaleSeconds } : {}),
+                    sources,
                 };
             }
         }
@@ -431,7 +502,7 @@ function validateConfigSchema(raw) {
         index: index,
         capabilities: capabilities,
         bundles: bundles,
-        bundle_imports,
+        subgraphs,
         pack: pack,
         templates: templates,
         work: work,

package/dist/graph/capabilities_index_cache.js

@@ -9,6 +9,7 @@ exports.loadCapabilitiesIndex = loadCapabilitiesIndex;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const paths_1 = require("../core/paths");
+const atomic_1 = require("../util/atomic");
 const workspace_files_1 = require("./workspace_files");
 const capabilities_indexer_1 = require("./capabilities_indexer");
 function mtimeMs(filePath) {
@@ -76,8 +77,7 @@ function readCapabilitiesIndex(indexPath) {
     }
 }
 function writeCapabilitiesIndex(indexPath, index) {
-    fs_1.default.mkdirSync(path_1.default.dirname(indexPath), { recursive: true });
-    fs_1.default.writeFileSync(indexPath, JSON.stringify(index, null, 2), "utf8");
+    (0, atomic_1.atomicWriteFile)(indexPath, JSON.stringify(index, null, 2));
 }
 function loadCapabilitiesIndex(options) {
     const useCache = options.useCache ?? true;

package/dist/graph/index_cache.js

@@ -8,9 +8,10 @@ exports.loadIndex = loadIndex;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const sort_1 = require("../util/sort");
+const atomic_1 = require("../util/atomic");
 const indexer_1 = require("./indexer");
 const staleness_1 = require("./staleness");
-const bundle_imports_1 = require("./bundle_imports");
+const subgraphs_1 = require("./subgraphs");
 function readIndex(indexPath) {
     try {
         const raw = fs_1.default.readFileSync(indexPath, "utf8");
@@ -23,8 +24,7 @@ function readIndex(indexPath) {
 }
 function writeIndex(indexPath, index) {
     const sortedIndex = { ...index, nodes: (0, sort_1.sortIndexNodes)(index.nodes) };
-    fs_1.default.mkdirSync(path_1.default.dirname(indexPath), { recursive: true });
-    fs_1.default.writeFileSync(indexPath, JSON.stringify(sortedIndex, null, 2));
+    (0, atomic_1.atomicWriteFile)(indexPath, JSON.stringify(sortedIndex, null, 2));
 }
 function loadIndex(options) {
     const useCache = options.useCache ?? true;
@@ -32,36 +32,36 @@ function loadIndex(options) {
     const tolerant = options.tolerant ?? options.config.index.tolerant;
     const includeImports = options.includeImports ?? true;
     const indexPath = path_1.default.resolve(options.root, options.config.index.global_index_path);
-    const withImports = (index, rebuilt, stale) => {
-        if (!includeImports || Object.keys(options.config.bundle_imports).length === 0) {
+    const withSubgraphs = (index, rebuilt, stale) => {
+        if (!includeImports || Object.keys(options.config.subgraphs).length === 0) {
             return { index, rebuilt, stale, warnings: [] };
         }
-        const imports = (0, bundle_imports_1.buildBundleImportsIndex)(options.root, options.config);
+        const subgraphs = (0, subgraphs_1.buildSubgraphsIndex)(options.root, options.config);
         if (allowReindex) {
-            (0, bundle_imports_1.writeBundleImportsIndex)((0, bundle_imports_1.resolveBundleImportsIndexPath)(options.root), imports.index);
+            (0, subgraphs_1.writeSubgraphsIndex)((0, subgraphs_1.resolveSubgraphsIndexPath)(options.root), subgraphs.index);
         }
         return {
-            index: (0, bundle_imports_1.mergeBundleImportsIntoIndex)(index, imports),
+            index: (0, subgraphs_1.mergeSubgraphsIntoIndex)(index, subgraphs),
             rebuilt,
-            stale: stale || (0, bundle_imports_1.isBundleImportsIndexStale)(options.root, options.config),
-            warnings: (0, bundle_imports_1.importWarnings)(imports),
+            stale: stale || (0, subgraphs_1.isSubgraphsIndexStale)(options.root, options.config),
+            warnings: (0, subgraphs_1.subgraphWarnings)(subgraphs),
         };
     };
     if (!useCache) {
         const index = (0, indexer_1.buildIndex)(options.root, options.config, { tolerant });
-        return withImports(index, true, false);
+        return withSubgraphs(index, true, false);
     }
     const stale = (0, staleness_1.isIndexStale)(options.root, options.config);
     if (fs_1.default.existsSync(indexPath) && !stale) {
-        return withImports(readIndex(indexPath), false, false);
+        return withSubgraphs(readIndex(indexPath), false, false);
     }
     if (allowReindex) {
         const index = (0, indexer_1.buildIndex)(options.root, options.config, { tolerant });
         writeIndex(indexPath, index);
-        return withImports(index, true, stale);
+        return withSubgraphs(index, true, stale);
     }
     if (fs_1.default.existsSync(indexPath)) {
-        return withImports(readIndex(indexPath), false, true);
+        return withSubgraphs(readIndex(indexPath), false, true);
     }
     throw new Error("index missing and auto-reindex is disabled");
 }

package/dist/graph/indexer.js

@@ -143,7 +143,7 @@ function buildIndex(root, config, options = {}) {
     };
     (0, validate_graph_1.validateGraph)(index, {
         allowMissing: tolerant,
-        externalWorkspaces: new Set(Object.keys(config.bundle_imports ?? {})),
+        externalWorkspaces: new Set(Object.keys(config.subgraphs ?? {})),
     });
     const latestCheckpointByWorkspace = {};
     for (const alias of workspaceAliases) {

package/dist/graph/reindex.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeDerivedIndexes = writeDerivedIndexes;
+const path_1 = __importDefault(require("path"));
+const capabilities_indexer_1 = require("./capabilities_indexer");
+const capabilities_index_cache_1 = require("./capabilities_index_cache");
+const subgraphs_1 = require("./subgraphs");
+const indexer_1 = require("./indexer");
+const index_cache_1 = require("./index_cache");
+const skills_index_cache_1 = require("./skills_index_cache");
+const skills_indexer_1 = require("./skills_indexer");
+const sqlite_index_1 = require("./sqlite_index");
+function writeDerivedIndexes(root, config, nodeIndex, options) {
+    const nextNodeIndex = nodeIndex ?? (0, indexer_1.buildIndex)(root, config, { tolerant: options?.tolerant ?? config.index.tolerant });
+    const skillsIndex = (0, skills_indexer_1.buildSkillsIndex)(root, config);
+    const capabilitiesIndex = (0, capabilities_indexer_1.buildCapabilitiesIndex)(root, config, nextNodeIndex);
+    const subgraphsIndex = (0, subgraphs_1.buildSubgraphsIndex)(root, config);
+    const nodesOutputPath = path_1.default.resolve(root, config.index.global_index_path);
+    const skillsOutputPath = (0, skills_indexer_1.resolveSkillsIndexPath)(root);
+    const capabilitiesOutputPath = (0, capabilities_indexer_1.resolveCapabilitiesIndexPath)(root, config);
+    const subgraphsOutputPath = (0, subgraphs_1.resolveSubgraphsIndexPath)(root);
+    (0, index_cache_1.writeIndex)(nodesOutputPath, nextNodeIndex);
+    (0, skills_index_cache_1.writeSkillsIndex)(skillsOutputPath, skillsIndex);
+    (0, capabilities_index_cache_1.writeCapabilitiesIndex)(capabilitiesOutputPath, capabilitiesIndex);
+    (0, subgraphs_1.writeSubgraphsIndex)(subgraphsOutputPath, subgraphsIndex.index);
+    const paths = {
+        nodes: nodesOutputPath,
+        skills: skillsOutputPath,
+        capabilities: capabilitiesOutputPath,
+        subgraphs: subgraphsOutputPath,
+    };
+    if ((0, sqlite_index_1.isSqliteBackend)(config)) {
+        paths.sqlite = (0, sqlite_index_1.writeSqliteIndex)({
+            root,
+            config,
+            nodeIndex: nextNodeIndex,
+            skillsIndex,
+            capabilitiesIndex,
+            subgraphsIndex: subgraphsIndex.index,
+        });
+    }
+    return { nodeIndex: nextNodeIndex, paths };
+}

package/dist/graph/skills_index_cache.js

@@ -8,6 +8,7 @@ exports.loadSkillsIndex = loadSkillsIndex;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const paths_1 = require("../core/paths");
+const atomic_1 = require("../util/atomic");
 const skills_indexer_1 = require("./skills_indexer");
 function mtimeMs(filePath) {
     return fs_1.default.statSync(filePath).mtimeMs;
@@ -67,8 +68,7 @@ function writeSkillsIndex(indexPath, index) {
         ...index,
         skills: sortedSkills,
     };
-    fs_1.default.mkdirSync(path_1.default.dirname(indexPath), { recursive: true });
-    fs_1.default.writeFileSync(indexPath, JSON.stringify(sortedIndex, null, 2), "utf8");
+    (0, atomic_1.atomicWriteFile)(indexPath, JSON.stringify(sortedIndex, null, 2));
 }
 function loadSkillsIndex(options) {
     const useCache = options.useCache ?? true;