@mars167/git-ai 2.3.0

package/dist/src/core/dsr/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/core/embedding/fusion.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WeightedEmbeddingFusion = void 0;
+exports.defaultFusionConfig = defaultFusionConfig;
+function normalize(vec) {
+    let norm = 0;
+    for (const v of vec)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm <= 0)
+        return vec.slice();
+    return vec.map((v) => v / norm);
+}
+function scale(vec, weight, targetDim) {
+    const out = new Array(targetDim).fill(0);
+    const len = Math.min(vec.length, targetDim);
+    for (let i = 0; i < len; i++)
+        out[i] = vec[i] * weight;
+    return out;
+}
+class WeightedEmbeddingFusion {
+    constructor(config) {
+        this.config = config;
+    }
+    fuse(semantic, structural, symbolic) {
+        const dim = Math.max(semantic.length, structural.length, symbolic.length);
+        const out = new Array(dim).fill(0);
+        const s0 = scale(semantic, this.config.semanticWeight, dim);
+        const s1 = scale(structural, this.config.structuralWeight, dim);
+        const s2 = scale(symbolic, this.config.symbolicWeight, dim);
+        for (let i = 0; i < dim; i++)
+            out[i] = s0[i] + s1[i] + s2[i];
+        return this.config.normalize ? normalize(out) : out;
+    }
+    fuseBatch(semantic, structural, symbolic) {
+        const count = Math.max(semantic.length, structural.length, symbolic.length);
+        const out = [];
+        for (let i = 0; i < count; i++) {
+            out.push(this.fuse(semantic[i] ?? [], structural[i] ?? [], symbolic[i] ?? []));
+        }
+        return out;
+    }
+}
+exports.WeightedEmbeddingFusion = WeightedEmbeddingFusion;
+function defaultFusionConfig() {
+    return {
+        semanticWeight: 0.5,
+        structuralWeight: 0.3,
+        symbolicWeight: 0.2,
+        normalize: true,
+    };
+}

package/dist/src/core/embedding/index.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HybridEmbedder = void 0;
+exports.defaultHybridEmbeddingConfig = defaultHybridEmbeddingConfig;
+const semantic_1 = require("./semantic");
+const structural_1 = require("./structural");
+const symbolic_1 = require("./symbolic");
+const fusion_1 = require("./fusion");
+const parser_1 = require("./parser");
+class HybridEmbedder {
+    constructor(config) {
+        this.config = config;
+        this.semantic = new semantic_1.OnnxSemanticEmbedder(config.semantic);
+        this.structural = new structural_1.WlStructuralEmbedder(config.structural);
+        this.symbolic = new symbolic_1.GraphSymbolicEmbedder(config.symbolic);
+        this.fusion = new fusion_1.WeightedEmbeddingFusion(config.fusion);
+    }
+    async embed(code, symbols) {
+        const [semanticVec] = await this.semantic.embedBatch([code]);
+        const structuralVec = this.structural.embed(this.parse(code));
+        const symbolicVec = this.symbolic.embedSymbols(symbols ?? []);
+        return this.fusion.fuse(semanticVec ?? [], structuralVec, symbolicVec);
+    }
+    async embedBatch(codes, symbols) {
+        const semanticVecs = await this.semantic.embedBatch(codes);
+        const structuralVecs = codes.map((code) => this.structural.embed(this.parse(code)));
+        const symbolicVecs = (symbols ?? []).map((s) => this.symbolic.embedSymbols(s ?? []));
+        const paddedSymbolic = codes.map((_, idx) => symbolicVecs[idx] ?? this.symbolic.embedSymbols([]));
+        return this.fusion.fuseBatch(semanticVecs, structuralVecs, paddedSymbolic);
+    }
+    parse(code) {
+        return (0, parser_1.parseCodeToTree)(code);
+    }
+}
+exports.HybridEmbedder = HybridEmbedder;
+function defaultHybridEmbeddingConfig() {
+    return {
+        semantic: (0, semantic_1.defaultSemanticConfig)(),
+        structural: (0, structural_1.defaultStructuralConfig)(),
+        symbolic: (0, symbolic_1.defaultSymbolicConfig)(),
+        fusion: (0, fusion_1.defaultFusionConfig)(),
+    };
+}

package/dist/src/core/embedding/parser.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCodeToTree = parseCodeToTree;
+const tree_sitter_1 = __importDefault(require("tree-sitter"));
+const typescript_1 = require("../parser/typescript");
+const adapter = new typescript_1.TypeScriptAdapter(false);
+function parseCodeToTree(code) {
+    const parser = new tree_sitter_1.default();
+    parser.setLanguage(adapter.getTreeSitterLanguage());
+    return parser.parse(code ?? '');
+}

package/dist/src/core/embedding/semantic.js

@@ -0,0 +1,254 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OnnxSemanticEmbedder = void 0;
+exports.defaultSemanticConfig = defaultSemanticConfig;
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const embedding_1 = require("../embedding");
+const crypto_1 = require("../crypto");
+const log_1 = require("../log");
+const log = (0, log_1.createLogger)({ component: 'embedding', kind: 'semantic' });
+class LruCache {
+    constructor(maxSize) {
+        this.maxSize = Math.max(1, maxSize);
+        this.map = new Map();
+    }
+    get(key) {
+        const value = this.map.get(key);
+        if (!value)
+            return undefined;
+        this.map.delete(key);
+        this.map.set(key, value);
+        return value;
+    }
+    set(key, value) {
+        if (this.map.has(key))
+            this.map.delete(key);
+        this.map.set(key, value);
+        if (this.map.size > this.maxSize) {
+            const first = this.map.keys().next().value;
+            if (first)
+                this.map.delete(first);
+        }
+    }
+}
+function normalize(vec) {
+    let norm = 0;
+    for (const v of vec)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm <= 0)
+        return vec.slice();
+    return vec.map((v) => v / norm);
+}
+function meanPool(hidden, attention, dims) {
+    const [batch, seqLen, dim] = dims;
+    if (batch !== 1)
+        throw new Error('meanPool expects batch=1');
+    const out = new Float32Array(dim);
+    let count = 0;
+    for (let i = 0; i < seqLen; i++) {
+        const att = Number(attention[i] ?? 0);
+        if (att === 0)
+            continue;
+        const offset = i * dim;
+        for (let d = 0; d < dim; d++)
+            out[d] += hidden[offset + d];
+        count += 1;
+    }
+    if (count === 0)
+        return Array.from(out);
+    for (let d = 0; d < dim; d++)
+        out[d] /= count;
+    return Array.from(out);
+}
+function padBigInt(values, target, pad = 0n) {
+    if (values.length >= target)
+        return values.slice(0, target);
+    const out = values.slice();
+    while (out.length < target)
+        out.push(pad);
+    return out;
+}
+function findModelPath(modelName) {
+    const resolved = path_1.default.isAbsolute(modelName) ? modelName : path_1.default.join(process.cwd(), modelName);
+    const candidates = [
+        resolved,
+        path_1.default.join(resolved, 'model.onnx'),
+        path_1.default.join(resolved, 'onnx', 'model.onnx'),
+    ];
+    for (const c of candidates) {
+        if (fs_extra_1.default.pathExistsSync(c))
+            return c;
+    }
+    return resolved;
+}
+async function loadOnnx() {
+    const moduleName = 'onnxruntime-node';
+    const mod = await import(moduleName);
+    return mod;
+}
+async function loadTokenizerModule() {
+    const moduleName = './tokenizer.js';
+    const mod = await import(moduleName);
+    return mod;
+}
+class OnnxSemanticEmbedder {
+    constructor(config) {
+        this.config = config;
+        this.cache = new LruCache(512);
+        this.onnxPromise = null;
+        this.sessionPromise = null;
+        this.tokenizerPromise = null;
+    }
+    async embed(code) {
+        const batch = await this.embedBatch([code]);
+        return batch[0] ?? new Array(this.config.embeddingDim).fill(0);
+    }
+    async embedBatch(codes) {
+        const clean = codes.map((c) => String(c ?? ''));
+        const results = new Array(clean.length);
+        const pending = [];
+        for (let i = 0; i < clean.length; i++) {
+            const key = (0, crypto_1.sha256Hex)(clean[i]);
+            const cached = this.cache.get(key);
+            if (cached) {
+                results[i] = cached.slice();
+            }
+            else {
+                pending.push({ index: i, code: clean[i], key });
+            }
+        }
+        if (pending.length === 0)
+            return results;
+        try {
+            const session = await this.getSession();
+            const tokenizer = await this.getTokenizer();
+            const batchSize = Math.max(1, this.config.batchSize);
+            for (let i = 0; i < pending.length; i += batchSize) {
+                const slice = pending.slice(i, i + batchSize);
+                const encoded = slice.map((item) => tokenizer.encode(item.code, { maxLength: 512 }));
+                const maxLen = Math.max(2, Math.min(512, Math.max(...encoded.map((e) => e.input_ids.length))));
+                const inputIds = encoded.map((e) => padBigInt(e.input_ids, maxLen, 0n));
+                const attentionMask = encoded.map((e) => padBigInt(e.attention_mask, maxLen, 0n));
+                const feeds = await this.buildFeeds(inputIds, attentionMask, maxLen, session);
+                const outputs = await session.run(feeds);
+                const outputName = Object.keys(outputs)[0];
+                const output = outputs[outputName];
+                if (!output)
+                    throw new Error('ONNX output missing');
+                const outputDims = output.dims;
+                const seqLen = outputDims?.[1] ?? maxLen;
+                const hiddenDim = outputDims?.[2] ?? this.config.embeddingDim;
+                const data = output.data;
+                const batchOut = [];
+                for (let b = 0; b < slice.length; b++) {
+                    const offset = b * seqLen * hiddenDim;
+                    const chunk = data.slice(offset, offset + seqLen * hiddenDim);
+                    const pooled = meanPool(chunk, attentionMask[b], [1, seqLen, hiddenDim]);
+                    const normalized = normalize(pooled);
+                    batchOut.push(this.ensureDim(normalized, this.config.embeddingDim));
+                }
+                for (let j = 0; j < slice.length; j++) {
+                    const out = batchOut[j] ?? new Array(this.config.embeddingDim).fill(0);
+                    results[slice[j].index] = out;
+                    this.cache.set(slice[j].key, out);
+                }
+            }
+            return results;
+        }
+        catch (err) {
+            log.warn('semantic_embed_fallback', { err: String(err?.message ?? err) });
+            for (const item of pending) {
+                const out = this.hashEmbed(item.code);
+                results[item.index] = out;
+                this.cache.set(item.key, out);
+            }
+            return results;
+        }
+    }
+    async getSession() {
+        if (!this.sessionPromise) {
+            this.sessionPromise = (async () => {
+                const onnx = await this.getOnnx();
+                const modelPath = findModelPath(this.config.modelName);
+                const providers = this.config.device === 'gpu' ? ['cuda', 'cpu'] : ['cpu'];
+                const opts = { executionProviders: providers };
+                const session = await onnx.InferenceSession.create(modelPath, opts);
+                log.info('semantic_session_ready', { model: modelPath, device: this.config.device });
+                return session;
+            })();
+        }
+        return this.sessionPromise;
+    }
+    async getTokenizer() {
+        if (!this.tokenizerPromise) {
+            this.tokenizerPromise = (async () => {
+                const mod = await loadTokenizerModule();
+                return mod.loadTokenizer(this.config.modelName);
+            })();
+        }
+        return this.tokenizerPromise;
+    }
+    async getOnnx() {
+        if (!this.onnxPromise)
+            this.onnxPromise = loadOnnx();
+        return this.onnxPromise;
+    }
+    async buildFeeds(inputIds, attentionMask, maxLen, session) {
+        const onnx = await this.getOnnx();
+        const batch = inputIds.length;
+        const flattenIds = inputIds.flat();
+        const flattenMask = attentionMask.flat();
+        const idsTensor = new onnx.Tensor('int64', BigInt64Array.from(flattenIds), [batch, maxLen]);
+        const maskTensor = new onnx.Tensor('int64', BigInt64Array.from(flattenMask), [batch, maxLen]);
+        const feeds = {};
+        const inputNames = ['input_ids', 'attention_mask', 'token_type_ids'];
+        for (const name of inputNames) {
+            if (name === 'input_ids')
+                feeds[name] = idsTensor;
+            if (name === 'attention_mask')
+                feeds[name] = maskTensor;
+            if (name === 'token_type_ids') {
+                const types = new onnx.Tensor('int64', new BigInt64Array(batch * maxLen), [batch, maxLen]);
+                feeds[name] = types;
+            }
+        }
+        return feeds;
+    }
+    ensureDim(vec, dim) {
+        if (vec.length === dim)
+            return vec;
+        if (vec.length > dim)
+            return vec.slice(0, dim);
+        const out = vec.slice();
+        while (out.length < dim)
+            out.push(0);
+        return out;
+    }
+    hashEmbed(text) {
+        const out = (0, embedding_1.hashEmbedding)(text, { dim: this.config.embeddingDim });
+        return out;
+    }
+}
+exports.OnnxSemanticEmbedder = OnnxSemanticEmbedder;
+function defaultSemanticConfig() {
+    // Support environment variable override
+    const modelPath = process.env.GIT_AI_EMBEDDING_MODEL ||
+        path_1.default.join(os_1.default.homedir(), '.cache', 'git-ai', 'models', 'codebert', 'model.onnx');
+    // Auto-detect embedding dimension based on model path
+    let embeddingDim = 768; // Default for CodeBERT
+    if (modelPath.includes('MiniLM')) {
+        embeddingDim = 384; // MiniLM-L6 uses 384 dimensions
+    }
+    return {
+        modelName: modelPath,
+        embeddingDim,
+        device: 'cpu',
+        batchSize: 4,
+    };
+}

package/dist/src/core/embedding/structural.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WlStructuralEmbedder = void 0;
+exports.defaultStructuralConfig = defaultStructuralConfig;
+const crypto_1 = require("../crypto");
+function normalize(vec) {
+    let norm = 0;
+    for (const v of vec)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm <= 0)
+        return vec.slice();
+    return vec.map((v) => v / norm);
+}
+function hashToIndex(hash, dim) {
+    const idx = parseInt(hash.slice(0, 8), 16) >>> 0;
+    return idx % dim;
+}
+function nodeFeatures(node) {
+    const childTypes = [];
+    for (let i = 0; i < node.namedChildCount; i++) {
+        const child = node.namedChild(i);
+        if (child)
+            childTypes.push(child.type);
+    }
+    let depth = 0;
+    let current = node;
+    while (current) {
+        depth += 1;
+        current = current.parent;
+    }
+    return { type: node.type, childTypes, depth };
+}
+function wlHash(type, neighborHashes, iteration) {
+    const base = [type, iteration.toString(), ...neighborHashes.sort()].join('|');
+    return (0, crypto_1.sha256Hex)(base);
+}
+class WlStructuralEmbedder {
+    constructor(config) {
+        this.config = config;
+    }
+    embed(tree) {
+        return this.embedNode(tree.rootNode);
+    }
+    embedNode(node) {
+        return this.embedSubtree(node);
+    }
+    embedSubtree(node) {
+        const dim = this.config.dim;
+        const iterations = Math.max(1, this.config.wlIterations);
+        const nodes = [];
+        const traverse = (n) => {
+            nodes.push(n);
+            for (let i = 0; i < n.namedChildCount; i++) {
+                const child = n.namedChild(i);
+                if (child)
+                    traverse(child);
+            }
+        };
+        traverse(node);
+        const currentHashes = new Map();
+        for (const n of nodes) {
+            const features = nodeFeatures(n);
+            const base = [features.type, features.childTypes.join(','), features.depth.toString()].join('|');
+            currentHashes.set(n, (0, crypto_1.sha256Hex)(base));
+        }
+        for (let iter = 0; iter < iterations; iter++) {
+            const next = new Map();
+            for (const n of nodes) {
+                const neighborHashes = [];
+                for (let i = 0; i < n.namedChildCount; i++) {
+                    const child = n.namedChild(i);
+                    if (child)
+                        neighborHashes.push(currentHashes.get(child) ?? '');
+                }
+                next.set(n, wlHash(n.type, neighborHashes, iter));
+            }
+            for (const [n, h] of next.entries())
+                currentHashes.set(n, h);
+        }
+        const vec = new Array(dim).fill(0);
+        for (const n of nodes) {
+            const h = currentHashes.get(n) ?? '';
+            const idx = hashToIndex(h, dim);
+            const sign = (parseInt(h.slice(0, 2), 16) & 1) === 0 ? 1 : -1;
+            vec[idx] += sign;
+            const features = nodeFeatures(n);
+            const depthIdx = (features.depth * 7) % dim;
+            vec[depthIdx] += 0.5;
+        }
+        return normalize(vec);
+    }
+}
+exports.WlStructuralEmbedder = WlStructuralEmbedder;
+function defaultStructuralConfig() {
+    return { dim: 256, wlIterations: 2 };
+}

package/dist/src/core/embedding/symbolic.js

@@ -0,0 +1,117 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GraphSymbolicEmbedder = void 0;
+exports.defaultSymbolicConfig = defaultSymbolicConfig;
+const crypto_1 = require("../crypto");
+function tokenize(text) {
+    const raw = String(text ?? '')
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/g)
+        .map((t) => t.trim())
+        .filter(Boolean);
+    const out = [];
+    for (const tok of raw) {
+        if (tok.length <= 6) {
+            out.push(tok);
+        }
+        else {
+            out.push(tok.slice(0, 3));
+            out.push(tok.slice(3, 6));
+            out.push(tok.slice(6));
+        }
+    }
+    return out;
+}
+function normalize(vec) {
+    let norm = 0;
+    for (const v of vec)
+        norm += v * v;
+    norm = Math.sqrt(norm);
+    if (norm <= 0)
+        return vec.slice();
+    return vec.map((v) => v / norm);
+}
+function hashToIndex(hash, dim) {
+    const idx = parseInt(hash.slice(0, 8), 16) >>> 0;
+    return idx % dim;
+}
+function addToken(vec, token, dim, weight) {
+    const hash = (0, crypto_1.sha256Hex)(token);
+    const idx = hashToIndex(hash, dim);
+    const sign = (parseInt(hash.slice(8, 10), 16) & 1) === 0 ? 1 : -1;
+    vec[idx] += sign * weight;
+}
+function addRelation(vec, a, b, dim, weight) {
+    const h = (0, crypto_1.sha256Hex)(`${a}=>${b}`);
+    const idx = hashToIndex(h, dim);
+    vec[idx] += weight;
+}
+class GraphSymbolicEmbedder {
+    constructor(config) {
+        this.config = config;
+    }
+    embedSymbols(symbols) {
+        const dim = this.config.dim;
+        const vec = new Array(dim).fill(0);
+        for (const sym of symbols) {
+            const nameTokens = tokenize(sym.name);
+            for (const t of nameTokens)
+                addToken(vec, t, dim, 1);
+            const signatureTokens = tokenize(sym.signature);
+            for (const t of signatureTokens)
+                addToken(vec, t, dim, 0.5);
+            addToken(vec, sym.kind, dim, 0.3);
+            if (sym.container) {
+                const containerTokens = tokenize(sym.container.name);
+                for (const t of containerTokens)
+                    addToken(vec, t, dim, 0.4);
+            }
+            if (sym.extends) {
+                for (const ext of sym.extends)
+                    addToken(vec, ext, dim, 0.6);
+            }
+            if (sym.implements) {
+                for (const iface of sym.implements)
+                    addToken(vec, iface, dim, 0.4);
+            }
+        }
+        return normalize(vec);
+    }
+    embedRelations(relations) {
+        const dim = this.config.dim;
+        const vec = new Array(dim).fill(0);
+        if (this.config.includeCalls) {
+            for (const [caller, callee] of relations.calls) {
+                addRelation(vec, caller, callee, dim, 1);
+                for (const t of tokenize(caller))
+                    addToken(vec, t, dim, 0.2);
+                for (const t of tokenize(callee))
+                    addToken(vec, t, dim, 0.2);
+            }
+        }
+        if (this.config.includeTypes) {
+            for (const [sub, sup] of relations.types) {
+                addRelation(vec, sub, sup, dim, 0.8);
+                for (const t of tokenize(sub))
+                    addToken(vec, t, dim, 0.15);
+                for (const t of tokenize(sup))
+                    addToken(vec, t, dim, 0.15);
+            }
+        }
+        if (this.config.includeImports) {
+            for (const [file, imp] of relations.imports) {
+                addRelation(vec, file, imp, dim, 0.6);
+            }
+        }
+        return normalize(vec);
+    }
+}
+exports.GraphSymbolicEmbedder = GraphSymbolicEmbedder;
+function defaultSymbolicConfig() {
+    return {
+        dim: 128,
+        includeCalls: true,
+        includeTypes: true,
+        includeImports: true,
+    };
+}

package/dist/src/core/embedding/tokenizer.js

@@ -0,0 +1,91 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTokenizer = loadTokenizer;
+const fs_extra_1 = __importDefault(require("fs-extra"));
+const path_1 = __importDefault(require("path"));
+class BasicTokenizer {
+    constructor(vocab, unkId, clsId, sepId) {
+        this.vocab = vocab;
+        this.unkId = unkId;
+        this.clsId = clsId;
+        this.sepId = sepId;
+    }
+    encode(text, options = {}) {
+        const maxLength = Math.max(2, Math.min(512, options.maxLength ?? 512));
+        const tokens = tokenize(text);
+        const ids = [BigInt(this.clsId)];
+        for (const tok of tokens) {
+            if (ids.length >= maxLength - 1)
+                break;
+            const id = this.vocab.get(tok) ?? this.unkId;
+            ids.push(BigInt(id));
+        }
+        ids.push(BigInt(this.sepId));
+        const attention = ids.map(() => 1n);
+        return { input_ids: ids, attention_mask: attention };
+    }
+}
+function tokenize(text) {
+    const raw = String(text ?? '')
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/g)
+        .map((t) => t.trim())
+        .filter(Boolean);
+    const out = [];
+    for (const tok of raw) {
+        if (tok.length <= 8) {
+            out.push(tok);
+        }
+        else {
+            out.push(tok.slice(0, 4));
+            out.push(tok.slice(4, 8));
+            out.push(tok.slice(8));
+        }
+    }
+    return out;
+}
+async function loadVocab(vocabPath) {
+    const vocab = new Map();
+    if (!await fs_extra_1.default.pathExists(vocabPath))
+        return vocab;
+    const content = await fs_extra_1.default.readFile(vocabPath, 'utf-8');
+    const lines = content.split(/\r?\n/).filter(Boolean);
+    for (let i = 0; i < lines.length; i++) {
+        vocab.set(lines[i].trim(), i);
+    }
+    return vocab;
+}
+async function loadTokenizer(modelName) {
+    const vocabCandidates = [
+        path_1.default.join(modelName, 'vocab.txt'),
+        path_1.default.join(modelName, 'tokenizer', 'vocab.txt'),
+        path_1.default.join(modelName, 'tokenizer', 'vocab.json'),
+    ];
+    let vocab = new Map();
+    for (const candidate of vocabCandidates) {
+        if (candidate.endsWith('vocab.json')) {
+            if (!await fs_extra_1.default.pathExists(candidate))
+                continue;
+            const json = await fs_extra_1.default.readJSON(candidate).catch(() => null);
+            if (json && typeof json === 'object') {
+                vocab = new Map();
+                for (const [key, value] of Object.entries(json)) {
+                    if (typeof value === 'number')
+                        vocab.set(key, value);
+                }
+            }
+        }
+        else {
+            vocab = await loadVocab(candidate);
+        }
+        if (vocab.size > 0)
+            break;
+    }
+    const unkId = vocab.get('[UNK]') ?? 100;
+    const clsId = vocab.get('[CLS]') ?? 101;
+    const sepId = vocab.get('[SEP]') ?? 102;
+    return new BasicTokenizer(vocab, unkId, clsId, sepId);
+}

package/dist/src/core/embedding/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });