@cesarandreslopez/occ 0.8.7 → 0.8.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -4
- package/dist/src/cli.d.ts +2 -1
- package/dist/src/cli.js +9 -6
- package/dist/src/cli.js.map +1 -1
- package/dist/src/code/build.js +90 -0
- package/dist/src/code/build.js.map +1 -1
- package/dist/src/code/command.js +49 -1
- package/dist/src/code/command.js.map +1 -1
- package/dist/src/code/isolated.js +7 -3
- package/dist/src/code/isolated.js.map +1 -1
- package/dist/src/code/languages.d.ts +3 -0
- package/dist/src/code/languages.js +90 -4
- package/dist/src/code/languages.js.map +1 -1
- package/dist/src/code/map.d.ts +145 -0
- package/dist/src/code/map.js +275 -0
- package/dist/src/code/map.js.map +1 -0
- package/dist/src/code/output.d.ts +5 -0
- package/dist/src/code/output.js +90 -0
- package/dist/src/code/output.js.map +1 -1
- package/dist/src/code/parsers.js +37 -13
- package/dist/src/code/parsers.js.map +1 -1
- package/dist/src/code/query.d.ts +40 -1
- package/dist/src/code/query.js +186 -1
- package/dist/src/code/query.js.map +1 -1
- package/dist/src/code/rank.d.ts +86 -0
- package/dist/src/code/rank.js +208 -0
- package/dist/src/code/rank.js.map +1 -0
- package/dist/src/code/session.d.ts +3 -1
- package/dist/src/code/session.js +7 -1
- package/dist/src/code/session.js.map +1 -1
- package/dist/src/code/slim.d.ts +10 -0
- package/dist/src/code/slim.js +10 -0
- package/dist/src/code/slim.js.map +1 -1
- package/dist/src/code/store.d.ts +4 -0
- package/dist/src/code/store.js +7 -0
- package/dist/src/code/store.js.map +1 -1
- package/dist/src/code/types.d.ts +90 -0
- package/dist/src/code/types.js +43 -0
- package/dist/src/code/types.js.map +1 -1
- package/dist/src/doc/chunk.d.ts +22 -0
- package/dist/src/doc/chunk.js +169 -0
- package/dist/src/doc/chunk.js.map +1 -0
- package/dist/src/doc/discover.d.ts +19 -3
- package/dist/src/doc/discover.js +35 -10
- package/dist/src/doc/discover.js.map +1 -1
- package/dist/src/errors.d.ts +6 -0
- package/dist/src/errors.js +15 -0
- package/dist/src/errors.js.map +1 -0
- package/dist/src/health.d.ts +12 -0
- package/dist/src/health.js +38 -0
- package/dist/src/health.js.map +1 -0
- package/dist/src/index.d.ts +34 -3
- package/dist/src/index.js +26 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/markdown/convert.js +17 -0
- package/dist/src/markdown/convert.js.map +1 -1
- package/dist/src/progress-event.d.ts +7 -1
- package/dist/src/tokens.d.ts +31 -0
- package/dist/src/tokens.js +88 -0
- package/dist/src/tokens.js.map +1 -1
- package/dist/src/utils.d.ts +1 -0
- package/dist/src/utils.js +16 -1
- package/dist/src/utils.js.map +1 -1
- package/dist/src/workspace/bundle.d.ts +67 -0
- package/dist/src/workspace/bundle.js +296 -0
- package/dist/src/workspace/bundle.js.map +1 -0
- package/dist/src/workspace/documents.d.ts +1 -0
- package/dist/src/workspace/documents.js +13 -18
- package/dist/src/workspace/documents.js.map +1 -1
- package/dist/src/workspace/prepare.js +7 -3
- package/dist/src/workspace/prepare.js.map +1 -1
- package/dist/src/workspace/types.d.ts +2 -2
- package/package.json +37 -1
|
@@ -1,13 +1,29 @@
|
|
|
1
1
|
export interface DiscoveredDocument {
|
|
2
2
|
filePath: string;
|
|
3
3
|
relativePath: string;
|
|
4
|
-
format:
|
|
4
|
+
format: DocumentFormat;
|
|
5
5
|
sizeBytes: number;
|
|
6
6
|
}
|
|
7
|
-
export
|
|
7
|
+
export interface DiscoveredDocumentSkippedEntry {
|
|
8
|
+
path: string;
|
|
9
|
+
reason: string;
|
|
10
|
+
size: number;
|
|
11
|
+
}
|
|
12
|
+
export interface DiscoverDocumentsResult {
|
|
13
|
+
documents: DiscoveredDocument[];
|
|
14
|
+
skipped: DiscoveredDocumentSkippedEntry[];
|
|
15
|
+
}
|
|
16
|
+
export declare const DOCUMENT_EXTENSIONS: readonly ["docx", "pdf", "pptx", "xlsx", "odt", "ods", "odp"];
|
|
17
|
+
export declare const PROSE_DOCUMENT_EXTENSIONS: readonly ["md", "markdown", "mdx", "txt", "rst", "adoc", "asciidoc"];
|
|
18
|
+
export declare const DATA_DOCUMENT_EXTENSIONS: readonly ["yaml", "yml", "json", "jsonc", "toml"];
|
|
19
|
+
export type DocumentFormat = typeof DOCUMENT_EXTENSIONS[number] | typeof PROSE_DOCUMENT_EXTENSIONS[number] | typeof DATA_DOCUMENT_EXTENSIONS[number];
|
|
20
|
+
export interface DiscoverDocumentsOptions {
|
|
8
21
|
excludeDir?: string[];
|
|
9
22
|
maxSizeBytes?: number;
|
|
10
23
|
noGitignore?: boolean;
|
|
11
24
|
ignorePatterns?: string[];
|
|
12
25
|
overlayIgnorePatterns?: string[];
|
|
13
|
-
|
|
26
|
+
includeDataFiles?: boolean;
|
|
27
|
+
}
|
|
28
|
+
export declare function discoverDocumentSet(rootDir: string, options?: DiscoverDocumentsOptions): Promise<DiscoverDocumentsResult>;
|
|
29
|
+
export declare function discoverDocuments(rootDir: string, options?: DiscoverDocumentsOptions): Promise<DiscoveredDocument[]>;
|
package/dist/src/doc/discover.js
CHANGED
|
@@ -2,11 +2,18 @@ import fg from 'fast-glob';
|
|
|
2
2
|
import { stat } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { buildExcludeDirGlobs, createIgnoreMatcher, normalizeRelativePath } from '../utils.js';
|
|
5
|
-
const DOCUMENT_EXTENSIONS = ['docx', 'pdf', 'pptx', 'xlsx', 'odt', 'ods', 'odp'];
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
export const DOCUMENT_EXTENSIONS = ['docx', 'pdf', 'pptx', 'xlsx', 'odt', 'ods', 'odp'];
|
|
6
|
+
export const PROSE_DOCUMENT_EXTENSIONS = ['md', 'markdown', 'mdx', 'txt', 'rst', 'adoc', 'asciidoc'];
|
|
7
|
+
export const DATA_DOCUMENT_EXTENSIONS = ['yaml', 'yml', 'json', 'jsonc', 'toml'];
|
|
8
|
+
function getDiscoverableExtensions(includeDataFiles) {
|
|
9
|
+
return includeDataFiles
|
|
10
|
+
? [...DOCUMENT_EXTENSIONS, ...PROSE_DOCUMENT_EXTENSIONS, ...DATA_DOCUMENT_EXTENSIONS]
|
|
11
|
+
: [...DOCUMENT_EXTENSIONS, ...PROSE_DOCUMENT_EXTENSIONS];
|
|
8
12
|
}
|
|
9
|
-
|
|
13
|
+
function isDocumentFormat(ext, extensions) {
|
|
14
|
+
return extensions.includes(ext);
|
|
15
|
+
}
|
|
16
|
+
export async function discoverDocumentSet(rootDir, options) {
|
|
10
17
|
const resolvedRoot = path.resolve(rootDir);
|
|
11
18
|
const excludeDir = options?.excludeDir ?? ['node_modules', '.git', 'dist', 'vendor', 'build', 'coverage', 'target'];
|
|
12
19
|
const maxSizeBytes = options?.maxSizeBytes ?? 50 * 1024 * 1024; // 50MB
|
|
@@ -16,7 +23,8 @@ export async function discoverDocuments(rootDir, options) {
|
|
|
16
23
|
ignorePatterns: options?.ignorePatterns,
|
|
17
24
|
overlayIgnorePatterns: options?.overlayIgnorePatterns,
|
|
18
25
|
});
|
|
19
|
-
const
|
|
26
|
+
const extensions = getDiscoverableExtensions(options?.includeDataFiles ?? false);
|
|
27
|
+
const pattern = `**/*.{${extensions.join(',')}}`;
|
|
20
28
|
const discovered = await fg(pattern, {
|
|
21
29
|
cwd: resolvedRoot,
|
|
22
30
|
absolute: true,
|
|
@@ -26,13 +34,22 @@ export async function discoverDocuments(rootDir, options) {
|
|
|
26
34
|
ignore: buildExcludeDirGlobs(excludeDir),
|
|
27
35
|
});
|
|
28
36
|
const documents = [];
|
|
37
|
+
const skipped = [];
|
|
29
38
|
for (const filePath of discovered.sort()) {
|
|
30
39
|
try {
|
|
31
40
|
const fileStat = await stat(filePath);
|
|
32
|
-
if (!fileStat.isFile()
|
|
41
|
+
if (!fileStat.isFile())
|
|
42
|
+
continue;
|
|
43
|
+
if (fileStat.size > maxSizeBytes) {
|
|
44
|
+
skipped.push({
|
|
45
|
+
path: filePath,
|
|
46
|
+
reason: `Exceeds ${Math.round(maxSizeBytes / 1024 / 1024)}MB limit`,
|
|
47
|
+
size: fileStat.size,
|
|
48
|
+
});
|
|
33
49
|
continue;
|
|
50
|
+
}
|
|
34
51
|
const ext = path.extname(filePath).toLowerCase().replace('.', '');
|
|
35
|
-
if (!isDocumentFormat(ext))
|
|
52
|
+
if (!isDocumentFormat(ext, extensions))
|
|
36
53
|
continue;
|
|
37
54
|
const relativePath = normalizeRelativePath(path.relative(resolvedRoot, filePath));
|
|
38
55
|
if (ignoreMatcher.ignores(relativePath))
|
|
@@ -44,10 +61,18 @@ export async function discoverDocuments(rootDir, options) {
|
|
|
44
61
|
sizeBytes: fileStat.size,
|
|
45
62
|
});
|
|
46
63
|
}
|
|
47
|
-
catch {
|
|
48
|
-
|
|
64
|
+
catch (err) {
|
|
65
|
+
const error = err;
|
|
66
|
+
skipped.push({
|
|
67
|
+
path: filePath,
|
|
68
|
+
reason: error.code === 'EACCES' ? 'Permission denied' : error.message,
|
|
69
|
+
size: 0,
|
|
70
|
+
});
|
|
49
71
|
}
|
|
50
72
|
}
|
|
51
|
-
return documents;
|
|
73
|
+
return { documents, skipped };
|
|
74
|
+
}
|
|
75
|
+
export async function discoverDocuments(rootDir, options) {
|
|
76
|
+
return (await discoverDocumentSet(rootDir, options)).documents;
|
|
52
77
|
}
|
|
53
78
|
//# sourceMappingURL=discover.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../src/doc/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../src/doc/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAoB/F,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AACjG,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAU,CAAC;AAC9G,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAU,CAAC;AAO1F,SAAS,yBAAyB,CAAC,gBAAyB;IAC1D,OAAO,gBAAgB;QACrB,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,yBAAyB,EAAE,GAAG,wBAAwB,CAAC;QACrF,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,yBAAyB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,UAAqC;IAC1E,OAAQ,UAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAe,EAAE,OAAkC;IAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpH,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;IACvE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE;QAC5D,UAAU;QACV,WAAW,EAAE,OAAO,EAAE,WAAW;QACjC,cAAc,EAAE,OAAO,EAAE,cAAc;QACvC,qBAAqB,EAAE,OAAO,EAAE,qBAAqB;KACtD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,yBAAyB,CAAC,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAEjD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE;QACnC,GAAG,EAAE,YAAY;QACjB,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,KAAK;QACV,mBAAmB,EAAE,KAAK;QAC1B,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC;KACzC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAyB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBAAE,SAAS;YACjC,IAAI,QAAQ,CAAC,IAAI,GAAG,YAAY,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,WAAW,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC,UAAU;oBACnE,IAAI,EAAE,QAAQ,CAAC,IAAI;iBACpB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,UAAU,CAAC;gBAAE,SAAS;YACjD,MAAM,YAAY,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClF,IAAI,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC;gBAAE,SAAS;YAElD,SAAS,CAAC,IAAI,CAAC;gBACb,QAAQ;gBACR,YAAY;gBACZ,MAAM,EAAE,GAAG;gBACX,SAAS,EAAE,QAAQ,CAAC,IAAI;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,GAA4B,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO;gBACrE,IAAI,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe,EAAE,OAAkC;IACzF,OAAO,CAAC,MAAM,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACjE,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export const OCC_ABORTED = 'OCC_ABORTED';
|
|
2
|
+
export class OccAbortError extends Error {
|
|
3
|
+
code = OCC_ABORTED;
|
|
4
|
+
constructor(message = 'The operation was aborted.') {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = 'AbortError';
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export function isOccAbortError(error) {
|
|
10
|
+
return error instanceof OccAbortError
|
|
11
|
+
|| (error instanceof Error
|
|
12
|
+
&& (error.name === 'AbortError'
|
|
13
|
+
|| error.code === OCC_ABORTED));
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;AAEzC,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,IAAI,GAAG,WAAW,CAAC;IAE5B,YAAY,OAAO,GAAG,4BAA4B;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;IAC3B,CAAC;CACF;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,KAAK,YAAY,aAAa;WAChC,CAAC,KAAK,YAAY,KAAK;eACrB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY;mBACzB,KAA4B,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
let cachedVersion;
|
|
5
|
+
async function readVersion() {
|
|
6
|
+
if (cachedVersion)
|
|
7
|
+
return cachedVersion;
|
|
8
|
+
const directory = path.dirname(fileURLToPath(import.meta.url));
|
|
9
|
+
for (const rel of ['..', '../..']) {
|
|
10
|
+
try {
|
|
11
|
+
const raw = await readFile(path.resolve(directory, rel, 'package.json'), 'utf8');
|
|
12
|
+
const parsed = JSON.parse(raw);
|
|
13
|
+
if (parsed.version) {
|
|
14
|
+
cachedVersion = parsed.version;
|
|
15
|
+
return parsed.version;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
// Try next package.json location.
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
cachedVersion = '0.0.0';
|
|
23
|
+
return cachedVersion;
|
|
24
|
+
}
|
|
25
|
+
export async function health() {
|
|
26
|
+
return {
|
|
27
|
+
available: true,
|
|
28
|
+
version: await readVersion(),
|
|
29
|
+
capabilities: {
|
|
30
|
+
code: true,
|
|
31
|
+
workspace: true,
|
|
32
|
+
documents: true,
|
|
33
|
+
tables: true,
|
|
34
|
+
health: true,
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=health.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"health.js","sourceRoot":"","sources":["../../src/health.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAczC,IAAI,aAAiC,CAAC;AAEtC,KAAK,UAAU,WAAW;IACxB,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;YACjF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;YACvD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC/B,OAAO,MAAM,CAAC,OAAO,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IACD,aAAa,GAAG,OAAO,CAAC;IACxB,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM;IAC1B,OAAO;QACL,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,MAAM,WAAW,EAAE;QAC5B,YAAY,EAAE;YACZ,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb;KACF,CAAC;AACJ,CAAC"}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -5,16 +5,23 @@ import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
|
|
|
5
5
|
import { discoverCodeFiles } from './code/discover.js';
|
|
6
6
|
import { previewCodebaseSize } from './code/preview.js';
|
|
7
7
|
import { buildCodebaseIndexIsolated } from './code/isolated.js';
|
|
8
|
-
import { parseSlimIndex, slimifyIndex, validateSlimIndex } from './code/slim.js';
|
|
8
|
+
import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
|
|
9
|
+
import { summarizeModule, toMermaid } from './code/query.js';
|
|
10
|
+
import { buildRepoMap } from './code/map.js';
|
|
11
|
+
import { rankNodes } from './code/rank.js';
|
|
9
12
|
import { inspectDocument } from './doc/inspect.js';
|
|
10
13
|
import { discoverDocuments } from './doc/discover.js';
|
|
14
|
+
import { chunkDocument } from './doc/chunk.js';
|
|
11
15
|
import { inspectWorkbook } from './sheet/inspect.js';
|
|
12
16
|
import { inspectPresentation } from './slide/inspect.js';
|
|
17
|
+
import { inspectTables } from './table/inspect.js';
|
|
13
18
|
import { analyzeWorkspace } from './workspace/analyze.js';
|
|
14
19
|
import { describeWorkspace } from './workspace/describe.js';
|
|
15
20
|
import { inspectWorkspaceDocumentSet } from './workspace/documents.js';
|
|
16
21
|
import { prepareWorkspaceContext } from './workspace/prepare.js';
|
|
17
|
-
|
|
22
|
+
import { bundleWorkspace } from './workspace/bundle.js';
|
|
23
|
+
import { health } from './health.js';
|
|
24
|
+
export type { CodebaseIndex, CodeNode, CodeEdge, CodeNodeType, CodeEdgeType, IndexTruncation, NormalizedSymbolKind } from './code/types.js';
|
|
18
25
|
export type { CodeQuerySession, CodeSessionChunkOptions } from './code/session.js';
|
|
19
26
|
export type { CodeIndexStore, CodeIndexStoreOptions, StoreStrategy, StoreMetadata, StoreSessionOptions } from './code/store.js';
|
|
20
27
|
export type { CodeChunk, CodeChunkOptions } from './code/chunk.js';
|
|
@@ -22,14 +29,26 @@ export type { BuildCodebaseOptions, CodeIndexBudget, BudgetExceededBehavior } fr
|
|
|
22
29
|
export type { BuildCodebaseIndexIsolatedOptions } from './code/isolated.js';
|
|
23
30
|
export type { CodebaseIndexSlim, SlimParsedFile } from './code/slim.js';
|
|
24
31
|
export type { CodebaseSizePreview, PreviewCodebaseSizeOptions } from './code/preview.js';
|
|
32
|
+
export type { ModuleSummary, ModuleSummaryOptions, MermaidKind, MermaidOptions } from './code/query.js';
|
|
33
|
+
export type { RepoMapOptions, RepoMapResult, RepoMapEntry, RepoMapSymbol, RepoMapFormat, RepoMapMode } from './code/map.js';
|
|
34
|
+
export type { RankOptions, RankResult, RankedNode, RankedFile, RankEdgeWeights } from './code/rank.js';
|
|
35
|
+
export type { DocumentChunk, DocumentChunkOptions } from './doc/chunk.js';
|
|
25
36
|
export type { OnProgress, ProgressEvent, ProgressPhase } from './progress-event.js';
|
|
26
37
|
export type { WorkspacePrepareOptions, WorkspacePreparedContext, WorkspacePreparationProgressEvent, OnPrepareProgress } from './workspace/prepare-types.js';
|
|
27
38
|
export type { AnalyzeWorkspaceOptions } from './workspace/analyze.js';
|
|
28
39
|
export type { DescribeWorkspaceOptions } from './workspace/describe.js';
|
|
29
40
|
export type { InspectWorkspaceDocumentSetOptions } from './workspace/documents.js';
|
|
41
|
+
export type { WorkspaceBundle, WorkspaceBundleOptions, WorkspaceOutlineNode, WorkspaceCodeDocumentReference } from './workspace/bundle.js';
|
|
30
42
|
export type { WorkspaceDescription, WorkspaceRecommendedCall } from './workspace/types.js';
|
|
31
|
-
export {
|
|
43
|
+
export type { OccHealth } from './health.js';
|
|
44
|
+
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
32
45
|
export { CodeIndexBudgetExceededError } from './code/build.js';
|
|
46
|
+
export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
|
|
47
|
+
export { health } from './health.js';
|
|
48
|
+
export { inspectTables } from './table/inspect.js';
|
|
49
|
+
export { chunkDocument } from './doc/chunk.js';
|
|
50
|
+
export { bundleWorkspace } from './workspace/bundle.js';
|
|
51
|
+
export { toNormalizedSymbolKind } from './code/types.js';
|
|
33
52
|
export interface OccCodeNamespace {
|
|
34
53
|
buildIndex: typeof buildCodebaseIndex;
|
|
35
54
|
buildIndexIsolated: typeof buildCodebaseIndexIsolated;
|
|
@@ -41,8 +60,13 @@ export interface OccCodeNamespace {
|
|
|
41
60
|
discover: typeof discoverCodeFiles;
|
|
42
61
|
chunk: typeof chunkCodebase;
|
|
43
62
|
chunkFromIndex: typeof chunkFromIndex;
|
|
63
|
+
summarizeModule: typeof summarizeModule;
|
|
64
|
+
toMermaid: typeof toMermaid;
|
|
65
|
+
map: typeof buildRepoMap;
|
|
66
|
+
rank: typeof rankNodes;
|
|
44
67
|
slimifyIndex: typeof slimifyIndex;
|
|
45
68
|
parseSlimIndex: typeof parseSlimIndex;
|
|
69
|
+
tryParseSlimIndex: typeof tryParseSlimIndex;
|
|
46
70
|
validateSlimIndex: typeof validateSlimIndex;
|
|
47
71
|
}
|
|
48
72
|
export interface OccWorkspaceNamespace {
|
|
@@ -50,10 +74,12 @@ export interface OccWorkspaceNamespace {
|
|
|
50
74
|
describe: typeof describeWorkspace;
|
|
51
75
|
inspectDocuments: typeof inspectWorkspaceDocumentSet;
|
|
52
76
|
prepare: typeof prepareWorkspaceContext;
|
|
77
|
+
bundle: typeof bundleWorkspace;
|
|
53
78
|
}
|
|
54
79
|
export interface OccDocNamespace {
|
|
55
80
|
inspect: typeof inspectDocument;
|
|
56
81
|
discover: typeof discoverDocuments;
|
|
82
|
+
chunk: typeof chunkDocument;
|
|
57
83
|
}
|
|
58
84
|
export interface OccSheetNamespace {
|
|
59
85
|
inspect: typeof inspectWorkbook;
|
|
@@ -61,11 +87,16 @@ export interface OccSheetNamespace {
|
|
|
61
87
|
export interface OccSlideNamespace {
|
|
62
88
|
inspect: typeof inspectPresentation;
|
|
63
89
|
}
|
|
90
|
+
export interface OccTableNamespace {
|
|
91
|
+
inspect: typeof inspectTables;
|
|
92
|
+
}
|
|
64
93
|
export interface OccNamespace {
|
|
65
94
|
code: OccCodeNamespace;
|
|
66
95
|
workspace: OccWorkspaceNamespace;
|
|
67
96
|
doc: OccDocNamespace;
|
|
68
97
|
sheet: OccSheetNamespace;
|
|
69
98
|
slide: OccSlideNamespace;
|
|
99
|
+
table: OccTableNamespace;
|
|
100
|
+
health: typeof health;
|
|
70
101
|
}
|
|
71
102
|
export declare function createOcc(): OccNamespace;
|
package/dist/src/index.js
CHANGED
|
@@ -5,17 +5,30 @@ import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
|
|
|
5
5
|
import { discoverCodeFiles } from './code/discover.js';
|
|
6
6
|
import { previewCodebaseSize } from './code/preview.js';
|
|
7
7
|
import { buildCodebaseIndexIsolated } from './code/isolated.js';
|
|
8
|
-
import { parseSlimIndex, slimifyIndex, validateSlimIndex } from './code/slim.js';
|
|
8
|
+
import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
|
|
9
|
+
import { summarizeModule, toMermaid } from './code/query.js';
|
|
10
|
+
import { buildRepoMap } from './code/map.js';
|
|
11
|
+
import { rankNodes } from './code/rank.js';
|
|
9
12
|
import { inspectDocument } from './doc/inspect.js';
|
|
10
13
|
import { discoverDocuments } from './doc/discover.js';
|
|
14
|
+
import { chunkDocument } from './doc/chunk.js';
|
|
11
15
|
import { inspectWorkbook } from './sheet/inspect.js';
|
|
12
16
|
import { inspectPresentation } from './slide/inspect.js';
|
|
17
|
+
import { inspectTables } from './table/inspect.js';
|
|
13
18
|
import { analyzeWorkspace } from './workspace/analyze.js';
|
|
14
19
|
import { describeWorkspace } from './workspace/describe.js';
|
|
15
20
|
import { inspectWorkspaceDocumentSet } from './workspace/documents.js';
|
|
16
21
|
import { prepareWorkspaceContext } from './workspace/prepare.js';
|
|
17
|
-
|
|
22
|
+
import { bundleWorkspace } from './workspace/bundle.js';
|
|
23
|
+
import { health } from './health.js';
|
|
24
|
+
export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
|
|
18
25
|
export { CodeIndexBudgetExceededError } from './code/build.js';
|
|
26
|
+
export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
|
|
27
|
+
export { health } from './health.js';
|
|
28
|
+
export { inspectTables } from './table/inspect.js';
|
|
29
|
+
export { chunkDocument } from './doc/chunk.js';
|
|
30
|
+
export { bundleWorkspace } from './workspace/bundle.js';
|
|
31
|
+
export { toNormalizedSymbolKind } from './code/types.js';
|
|
19
32
|
export function createOcc() {
|
|
20
33
|
return {
|
|
21
34
|
code: {
|
|
@@ -29,8 +42,13 @@ export function createOcc() {
|
|
|
29
42
|
discover: discoverCodeFiles,
|
|
30
43
|
chunk: chunkCodebase,
|
|
31
44
|
chunkFromIndex: chunkFromIndex,
|
|
45
|
+
summarizeModule,
|
|
46
|
+
toMermaid,
|
|
47
|
+
map: buildRepoMap,
|
|
48
|
+
rank: rankNodes,
|
|
32
49
|
slimifyIndex,
|
|
33
50
|
parseSlimIndex,
|
|
51
|
+
tryParseSlimIndex,
|
|
34
52
|
validateSlimIndex,
|
|
35
53
|
},
|
|
36
54
|
workspace: {
|
|
@@ -38,10 +56,12 @@ export function createOcc() {
|
|
|
38
56
|
describe: describeWorkspace,
|
|
39
57
|
inspectDocuments: inspectWorkspaceDocumentSet,
|
|
40
58
|
prepare: prepareWorkspaceContext,
|
|
59
|
+
bundle: bundleWorkspace,
|
|
41
60
|
},
|
|
42
61
|
doc: {
|
|
43
62
|
inspect: inspectDocument,
|
|
44
63
|
discover: discoverDocuments,
|
|
64
|
+
chunk: chunkDocument,
|
|
45
65
|
},
|
|
46
66
|
sheet: {
|
|
47
67
|
inspect: inspectWorkbook,
|
|
@@ -49,6 +69,10 @@ export function createOcc() {
|
|
|
49
69
|
slide: {
|
|
50
70
|
inspect: inspectPresentation,
|
|
51
71
|
},
|
|
72
|
+
table: {
|
|
73
|
+
inspect: inspectTables,
|
|
74
|
+
},
|
|
75
|
+
health,
|
|
52
76
|
};
|
|
53
77
|
}
|
|
54
78
|
//# sourceMappingURL=index.js.map
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,+BAA+B,GAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,+BAA+B,GAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAwBrC,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,YAAY,EACZ,SAAS,EACT,aAAa,EACb,cAAc,EACd,sBAAsB,EACtB,+BAA+B,EAC/B,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,IAAI,cAAc,EACzC,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,iBAAiB,EACjB,iBAAiB,GAClB,CAAC;AACF,OAAO,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AA2DzD,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,IAAI,EAAE;YACJ,UAAU,EAAE,kBAAkB;YAC9B,kBAAkB,EAAE,0BAA0B;YAC9C,aAAa,EAAE,sBAAsB;YACrC,sBAAsB,EAAE,+BAA+B;YACvD,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,uBAAuB;YACvC,WAAW,EAAE,mBAAmB;YAChC,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;YACpB,cAAc,EAAE,cAAc;YAC9B,eAAe;YACf,SAAS;YACT,GAAG,EAAE,YAAY;YACjB,IAAI,EAAE,SAAS;YACf,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,iBAAiB;SAClB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,gBAAgB;YACzB,QAAQ,EAAE,iBAAiB;YAC3B,gBAAgB,EAAE,2BAA2B;YAC7C,OAAO,EAAE,uBAAuB;YAChC,MAAM,EAAE,eAAe;SACxB;QACD,GAAG,EAAE;YACH,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;SACrB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,eAAe;SACzB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,mBAAmB;SAC7B;QACD,KAAK,EAAE;YACL,OAAO,EAAE,aAAa;SACvB;QACD,MAAM;KACP,CAAC;AACJ,CAAC"}
|
|
@@ -8,6 +8,20 @@ import { z } from 'zod';
|
|
|
8
8
|
import { extractOfficeText, getExtension } from '../utils.js';
|
|
9
9
|
const turndown = new TurndownService({ headingStyle: 'atx' });
|
|
10
10
|
const PdfTextContentSchema = z.object({ items: z.array(z.object({ str: z.string() }).passthrough()) });
|
|
11
|
+
const RAW_TEXT_EXTENSIONS = new Set([
|
|
12
|
+
'md',
|
|
13
|
+
'markdown',
|
|
14
|
+
'mdx',
|
|
15
|
+
'txt',
|
|
16
|
+
'rst',
|
|
17
|
+
'adoc',
|
|
18
|
+
'asciidoc',
|
|
19
|
+
'yaml',
|
|
20
|
+
'yml',
|
|
21
|
+
'json',
|
|
22
|
+
'jsonc',
|
|
23
|
+
'toml',
|
|
24
|
+
]);
|
|
11
25
|
/** Convert a DOCX file to markdown via mammoth → HTML → turndown */
|
|
12
26
|
async function docxToMarkdown(filePath) {
|
|
13
27
|
const result = await mammoth.convertToHtml({ path: filePath });
|
|
@@ -154,6 +168,9 @@ async function xlsxToMarkdown(filePath, maxRows = 20) {
|
|
|
154
168
|
/** Convert a document to markdown. Returns null for unsupported formats (ods). */
|
|
155
169
|
export async function documentToMarkdown(filePath) {
|
|
156
170
|
const ext = getExtension(filePath);
|
|
171
|
+
if (RAW_TEXT_EXTENSIONS.has(ext)) {
|
|
172
|
+
return readFile(filePath, 'utf8');
|
|
173
|
+
}
|
|
157
174
|
switch (ext) {
|
|
158
175
|
case 'docx': return docxToMarkdown(filePath);
|
|
159
176
|
case 'pdf': return pdfToMarkdown(filePath);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/markdown/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,GAAG,MAAM,WAAW,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAC9D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/markdown/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,GAAG,MAAM,WAAW,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAC9D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;AACvG,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,IAAI;IACJ,UAAU;IACV,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,UAAU;IACV,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC,CAAC;AAEH,oEAAoE;AACpE,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,sDAAsD;AACtD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;IAChC,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;QACnC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC;YACrJ,OAAO;QACT,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC;IAEF,IAAI,IAAwC,CAAC;IAC7C,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE;YACvB,UAAU,EAAE,KAAK,EAAE,QAAuE,EAAE,EAAE;gBAC5F,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;gBAChF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACxD,OAAO,SAAS,QAAQ,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,CAAC;SACF,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,GAAG,WAAW,CAAC;IAC5B,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC;AACnB,CAAC;AAED,oDAAoD;AACpD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE1C,uCAAuC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;SACtC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACzD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,OAAO,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IAEL,iCAAiC;IACjC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;IAErC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,gBAAgB,QAAQ,EAAE,CAAC;IACpC,CAAC;IAED,oCAAoC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,sEAAsE;AACtE,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAE3B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE7C,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,OAAO,gBAAgB,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,OAAO,GAAG,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,IAAI;QACf,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC;YAAE,SAAS;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE5C,KAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,kBAAkB;QAClB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrE,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;QAED,uBAAuB;QACvB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;YACjE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,GAAG,OAAO,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,GAAG,CAAC,GAAG,OAAO,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACvD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEnC,IAAI,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK;YACR,OAAO,IAAI,CAAC;QACd;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
export type ProgressPhase = 'start' | 'discover' | 'parse' | 'index' | 'graph' | 'serialize' | 'deserialize' | 'cache-check' | 'cache-load' | 'cache-save' | 'chunk' | 'inspect' | 'references' | 'cross-references' | 'discover-documents' | 'parse-documents' | 'structure' | 'code' | 'complete';
|
|
1
|
+
export type ProgressPhase = 'start' | 'bundle' | 'description' | 'stats' | 'code-preview' | 'discover' | 'parse' | 'index' | 'graph' | 'serialize' | 'deserialize' | 'cache-check' | 'cache-load' | 'cache-save' | 'chunk' | 'inspect' | 'references' | 'cross-references' | 'document-chunk' | 'discover-documents' | 'parse-documents' | 'structure' | 'outline' | 'code' | 'complete';
|
|
2
2
|
export interface ProgressEvent {
|
|
3
3
|
phase: ProgressPhase;
|
|
4
4
|
total: number;
|
|
5
5
|
completed: number;
|
|
6
6
|
detail?: string;
|
|
7
|
+
scope?: string;
|
|
8
|
+
currentPath?: string;
|
|
9
|
+
bytesProcessed?: number;
|
|
10
|
+
totalBytes?: number;
|
|
11
|
+
startedAt?: number;
|
|
12
|
+
elapsedMs?: number;
|
|
7
13
|
}
|
|
8
14
|
export type OnProgress = (event: ProgressEvent) => void;
|
package/dist/src/tokens.d.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Language-aware token estimation — tokenx-inspired approach.
|
|
3
3
|
* Replaces the naive `chars / 4` heuristic with segment-level analysis.
|
|
4
|
+
*
|
|
5
|
+
* The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
|
|
6
|
+
* zero-dependency default. For exact, model-specific counts, use the
|
|
7
|
+
* pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
|
|
8
|
+
* loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
|
|
9
|
+
* only when requested.
|
|
4
10
|
*/
|
|
11
|
+
import { z } from 'zod';
|
|
5
12
|
export interface LanguageConfig {
|
|
6
13
|
pattern: RegExp;
|
|
7
14
|
averageCharsPerToken: number;
|
|
@@ -14,3 +21,27 @@ export interface TokenEstimationOptions {
|
|
|
14
21
|
export declare function estimateTokenCount(text?: string | number, options?: TokenEstimationOptions): number;
|
|
15
22
|
/** Slice text by token positions (like Array.slice but for tokens) */
|
|
16
23
|
export declare function sliceByTokens(text: string, start?: number, end?: number, options?: TokenEstimationOptions): string;
|
|
24
|
+
export declare const TokenizerNameSchema: z.ZodEnum<{
|
|
25
|
+
heuristic: "heuristic";
|
|
26
|
+
o200k_base: "o200k_base";
|
|
27
|
+
cl100k_base: "cl100k_base";
|
|
28
|
+
}>;
|
|
29
|
+
export type TokenizerName = z.infer<typeof TokenizerNameSchema>;
|
|
30
|
+
export interface Tokenizer {
|
|
31
|
+
/** Identifier of the active encoding (`heuristic` for the language-aware estimate). */
|
|
32
|
+
readonly name: TokenizerName;
|
|
33
|
+
/** Token count for `text`. A number input is treated as a character count (heuristic only). */
|
|
34
|
+
count(text: string | number): number;
|
|
35
|
+
/** Slice `text` by token positions, like `Array.prototype.slice` but token-aware. */
|
|
36
|
+
slice(text: string, start?: number, end?: number): string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Build a tokenizer for the given encoding. `heuristic` resolves synchronously
|
|
40
|
+
* with no dependency; BPE encodings lazily load their tables once (cached).
|
|
41
|
+
*/
|
|
42
|
+
export declare function createTokenizer(name: TokenizerName): Promise<Tokenizer>;
|
|
43
|
+
/**
|
|
44
|
+
* Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
|
|
45
|
+
* env var, defaulting to `heuristic`. Throws on an unrecognized value.
|
|
46
|
+
*/
|
|
47
|
+
export declare function resolveTokenizerName(explicit?: string, env?: string | undefined): TokenizerName;
|
package/dist/src/tokens.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Language-aware token estimation — tokenx-inspired approach.
|
|
3
3
|
* Replaces the naive `chars / 4` heuristic with segment-level analysis.
|
|
4
|
+
*
|
|
5
|
+
* The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
|
|
6
|
+
* zero-dependency default. For exact, model-specific counts, use the
|
|
7
|
+
* pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
|
|
8
|
+
* loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
|
|
9
|
+
* only when requested.
|
|
4
10
|
*/
|
|
11
|
+
import { z } from 'zod';
|
|
5
12
|
const PATTERNS = {
|
|
6
13
|
whitespace: /^\s+$/,
|
|
7
14
|
cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
|
|
@@ -104,4 +111,85 @@ export function sliceByTokens(text, start = 0, end, options = {}) {
|
|
|
104
111
|
}
|
|
105
112
|
return parts.join('');
|
|
106
113
|
}
|
|
114
|
+
// ── Pluggable tokenizers ──────────────────────────────────────────────────
|
|
115
|
+
//
|
|
116
|
+
// `Tokenizer` lets callers swap the heuristic estimator for an exact BPE
|
|
117
|
+
// tokenizer without changing call sites: every consumer takes a
|
|
118
|
+
// `(text) => number` counter (or a `Tokenizer`) and threads it down. There is
|
|
119
|
+
// deliberately NO mutable module-level "current tokenizer" — the codebase
|
|
120
|
+
// keeps zero global state, so the selected tokenizer is always passed
|
|
121
|
+
// explicitly (resolved once at an entry point via `resolveTokenizerName` +
|
|
122
|
+
// `createTokenizer`).
|
|
123
|
+
export const TokenizerNameSchema = z.enum(['heuristic', 'o200k_base', 'cl100k_base']);
|
|
124
|
+
class HeuristicTokenizer {
|
|
125
|
+
name = 'heuristic';
|
|
126
|
+
count(text) {
|
|
127
|
+
return estimateTokenCount(text);
|
|
128
|
+
}
|
|
129
|
+
slice(text, start = 0, end) {
|
|
130
|
+
return sliceByTokens(text, start, end);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const encodingCache = new Map();
|
|
134
|
+
async function loadEncoding(name) {
|
|
135
|
+
const cached = encodingCache.get(name);
|
|
136
|
+
if (cached)
|
|
137
|
+
return cached;
|
|
138
|
+
const mod = name === 'o200k_base'
|
|
139
|
+
? await import('gpt-tokenizer/encoding/o200k_base')
|
|
140
|
+
: await import('gpt-tokenizer/encoding/cl100k_base');
|
|
141
|
+
encodingCache.set(name, mod);
|
|
142
|
+
return mod;
|
|
143
|
+
}
|
|
144
|
+
class BpeTokenizer {
|
|
145
|
+
name;
|
|
146
|
+
encoding;
|
|
147
|
+
constructor(name, encoding) {
|
|
148
|
+
this.name = name;
|
|
149
|
+
this.encoding = encoding;
|
|
150
|
+
}
|
|
151
|
+
count(text) {
|
|
152
|
+
// A bare character count has no text to encode; fall back to the heuristic.
|
|
153
|
+
if (typeof text === 'number')
|
|
154
|
+
return estimateTokenCount(text);
|
|
155
|
+
if (!text)
|
|
156
|
+
return 0;
|
|
157
|
+
return this.encoding.encode(text).length;
|
|
158
|
+
}
|
|
159
|
+
slice(text, start = 0, end) {
|
|
160
|
+
if (!text)
|
|
161
|
+
return '';
|
|
162
|
+
const tokens = this.encoding.encode(text);
|
|
163
|
+
const total = tokens.length;
|
|
164
|
+
const from = start < 0 ? Math.max(0, total + start) : Math.min(start, total);
|
|
165
|
+
const to = end === undefined ? total : end < 0 ? Math.max(0, total + end) : Math.min(end, total);
|
|
166
|
+
if (from >= to)
|
|
167
|
+
return '';
|
|
168
|
+
return this.encoding.decode(tokens.slice(from, to));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Build a tokenizer for the given encoding. `heuristic` resolves synchronously
|
|
173
|
+
* with no dependency; BPE encodings lazily load their tables once (cached).
|
|
174
|
+
*/
|
|
175
|
+
export async function createTokenizer(name) {
|
|
176
|
+
if (name === 'heuristic')
|
|
177
|
+
return new HeuristicTokenizer();
|
|
178
|
+
const encoding = await loadEncoding(name);
|
|
179
|
+
return new BpeTokenizer(name, encoding);
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
|
|
183
|
+
* env var, defaulting to `heuristic`. Throws on an unrecognized value.
|
|
184
|
+
*/
|
|
185
|
+
export function resolveTokenizerName(explicit, env = process.env.OCC_TOKENIZER) {
|
|
186
|
+
const candidate = explicit ?? env;
|
|
187
|
+
if (candidate == null || candidate === '')
|
|
188
|
+
return 'heuristic';
|
|
189
|
+
const result = TokenizerNameSchema.safeParse(candidate);
|
|
190
|
+
if (!result.success) {
|
|
191
|
+
throw new Error(`Invalid tokenizer "${candidate}". Expected one of: ${TokenizerNameSchema.options.join(', ')}`);
|
|
192
|
+
}
|
|
193
|
+
return result.data;
|
|
194
|
+
}
|
|
107
195
|
//# sourceMappingURL=tokens.js.map
|