@cesarandreslopez/occ 0.8.7 → 0.8.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +60 -4
  2. package/dist/src/cli.d.ts +2 -1
  3. package/dist/src/cli.js +9 -6
  4. package/dist/src/cli.js.map +1 -1
  5. package/dist/src/code/build.js +90 -0
  6. package/dist/src/code/build.js.map +1 -1
  7. package/dist/src/code/command.js +49 -1
  8. package/dist/src/code/command.js.map +1 -1
  9. package/dist/src/code/isolated.js +7 -3
  10. package/dist/src/code/isolated.js.map +1 -1
  11. package/dist/src/code/languages.d.ts +3 -0
  12. package/dist/src/code/languages.js +90 -4
  13. package/dist/src/code/languages.js.map +1 -1
  14. package/dist/src/code/map.d.ts +145 -0
  15. package/dist/src/code/map.js +275 -0
  16. package/dist/src/code/map.js.map +1 -0
  17. package/dist/src/code/output.d.ts +5 -0
  18. package/dist/src/code/output.js +90 -0
  19. package/dist/src/code/output.js.map +1 -1
  20. package/dist/src/code/parsers.js +37 -13
  21. package/dist/src/code/parsers.js.map +1 -1
  22. package/dist/src/code/query.d.ts +40 -1
  23. package/dist/src/code/query.js +186 -1
  24. package/dist/src/code/query.js.map +1 -1
  25. package/dist/src/code/rank.d.ts +86 -0
  26. package/dist/src/code/rank.js +208 -0
  27. package/dist/src/code/rank.js.map +1 -0
  28. package/dist/src/code/session.d.ts +3 -1
  29. package/dist/src/code/session.js +7 -1
  30. package/dist/src/code/session.js.map +1 -1
  31. package/dist/src/code/slim.d.ts +10 -0
  32. package/dist/src/code/slim.js +10 -0
  33. package/dist/src/code/slim.js.map +1 -1
  34. package/dist/src/code/store.d.ts +4 -0
  35. package/dist/src/code/store.js +7 -0
  36. package/dist/src/code/store.js.map +1 -1
  37. package/dist/src/code/types.d.ts +90 -0
  38. package/dist/src/code/types.js +43 -0
  39. package/dist/src/code/types.js.map +1 -1
  40. package/dist/src/doc/chunk.d.ts +22 -0
  41. package/dist/src/doc/chunk.js +169 -0
  42. package/dist/src/doc/chunk.js.map +1 -0
  43. package/dist/src/doc/discover.d.ts +19 -3
  44. package/dist/src/doc/discover.js +35 -10
  45. package/dist/src/doc/discover.js.map +1 -1
  46. package/dist/src/errors.d.ts +6 -0
  47. package/dist/src/errors.js +15 -0
  48. package/dist/src/errors.js.map +1 -0
  49. package/dist/src/health.d.ts +12 -0
  50. package/dist/src/health.js +38 -0
  51. package/dist/src/health.js.map +1 -0
  52. package/dist/src/index.d.ts +34 -3
  53. package/dist/src/index.js +26 -2
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/markdown/convert.js +17 -0
  56. package/dist/src/markdown/convert.js.map +1 -1
  57. package/dist/src/progress-event.d.ts +7 -1
  58. package/dist/src/tokens.d.ts +31 -0
  59. package/dist/src/tokens.js +88 -0
  60. package/dist/src/tokens.js.map +1 -1
  61. package/dist/src/utils.d.ts +1 -0
  62. package/dist/src/utils.js +16 -1
  63. package/dist/src/utils.js.map +1 -1
  64. package/dist/src/workspace/bundle.d.ts +67 -0
  65. package/dist/src/workspace/bundle.js +296 -0
  66. package/dist/src/workspace/bundle.js.map +1 -0
  67. package/dist/src/workspace/documents.d.ts +1 -0
  68. package/dist/src/workspace/documents.js +13 -18
  69. package/dist/src/workspace/documents.js.map +1 -1
  70. package/dist/src/workspace/prepare.js +7 -3
  71. package/dist/src/workspace/prepare.js.map +1 -1
  72. package/dist/src/workspace/types.d.ts +2 -2
  73. package/package.json +37 -1
@@ -1,13 +1,29 @@
1
1
  export interface DiscoveredDocument {
2
2
  filePath: string;
3
3
  relativePath: string;
4
- format: 'docx' | 'pdf' | 'pptx' | 'xlsx' | 'odt' | 'ods' | 'odp';
4
+ format: DocumentFormat;
5
5
  sizeBytes: number;
6
6
  }
7
- export declare function discoverDocuments(rootDir: string, options?: {
7
+ export interface DiscoveredDocumentSkippedEntry {
8
+ path: string;
9
+ reason: string;
10
+ size: number;
11
+ }
12
+ export interface DiscoverDocumentsResult {
13
+ documents: DiscoveredDocument[];
14
+ skipped: DiscoveredDocumentSkippedEntry[];
15
+ }
16
+ export declare const DOCUMENT_EXTENSIONS: readonly ["docx", "pdf", "pptx", "xlsx", "odt", "ods", "odp"];
17
+ export declare const PROSE_DOCUMENT_EXTENSIONS: readonly ["md", "markdown", "mdx", "txt", "rst", "adoc", "asciidoc"];
18
+ export declare const DATA_DOCUMENT_EXTENSIONS: readonly ["yaml", "yml", "json", "jsonc", "toml"];
19
+ export type DocumentFormat = typeof DOCUMENT_EXTENSIONS[number] | typeof PROSE_DOCUMENT_EXTENSIONS[number] | typeof DATA_DOCUMENT_EXTENSIONS[number];
20
+ export interface DiscoverDocumentsOptions {
8
21
  excludeDir?: string[];
9
22
  maxSizeBytes?: number;
10
23
  noGitignore?: boolean;
11
24
  ignorePatterns?: string[];
12
25
  overlayIgnorePatterns?: string[];
13
- }): Promise<DiscoveredDocument[]>;
26
+ includeDataFiles?: boolean;
27
+ }
28
+ export declare function discoverDocumentSet(rootDir: string, options?: DiscoverDocumentsOptions): Promise<DiscoverDocumentsResult>;
29
+ export declare function discoverDocuments(rootDir: string, options?: DiscoverDocumentsOptions): Promise<DiscoveredDocument[]>;
@@ -2,11 +2,18 @@ import fg from 'fast-glob';
2
2
  import { stat } from 'node:fs/promises';
3
3
  import path from 'node:path';
4
4
  import { buildExcludeDirGlobs, createIgnoreMatcher, normalizeRelativePath } from '../utils.js';
5
- const DOCUMENT_EXTENSIONS = ['docx', 'pdf', 'pptx', 'xlsx', 'odt', 'ods', 'odp'];
6
- function isDocumentFormat(ext) {
7
- return DOCUMENT_EXTENSIONS.includes(ext);
5
+ export const DOCUMENT_EXTENSIONS = ['docx', 'pdf', 'pptx', 'xlsx', 'odt', 'ods', 'odp'];
6
+ export const PROSE_DOCUMENT_EXTENSIONS = ['md', 'markdown', 'mdx', 'txt', 'rst', 'adoc', 'asciidoc'];
7
+ export const DATA_DOCUMENT_EXTENSIONS = ['yaml', 'yml', 'json', 'jsonc', 'toml'];
8
+ function getDiscoverableExtensions(includeDataFiles) {
9
+ return includeDataFiles
10
+ ? [...DOCUMENT_EXTENSIONS, ...PROSE_DOCUMENT_EXTENSIONS, ...DATA_DOCUMENT_EXTENSIONS]
11
+ : [...DOCUMENT_EXTENSIONS, ...PROSE_DOCUMENT_EXTENSIONS];
8
12
  }
9
- export async function discoverDocuments(rootDir, options) {
13
+ function isDocumentFormat(ext, extensions) {
14
+ return extensions.includes(ext);
15
+ }
16
+ export async function discoverDocumentSet(rootDir, options) {
10
17
  const resolvedRoot = path.resolve(rootDir);
11
18
  const excludeDir = options?.excludeDir ?? ['node_modules', '.git', 'dist', 'vendor', 'build', 'coverage', 'target'];
12
19
  const maxSizeBytes = options?.maxSizeBytes ?? 50 * 1024 * 1024; // 50MB
@@ -16,7 +23,8 @@ export async function discoverDocuments(rootDir, options) {
16
23
  ignorePatterns: options?.ignorePatterns,
17
24
  overlayIgnorePatterns: options?.overlayIgnorePatterns,
18
25
  });
19
- const pattern = `**/*.{${DOCUMENT_EXTENSIONS.join(',')}}`;
26
+ const extensions = getDiscoverableExtensions(options?.includeDataFiles ?? false);
27
+ const pattern = `**/*.{${extensions.join(',')}}`;
20
28
  const discovered = await fg(pattern, {
21
29
  cwd: resolvedRoot,
22
30
  absolute: true,
@@ -26,13 +34,22 @@ export async function discoverDocuments(rootDir, options) {
26
34
  ignore: buildExcludeDirGlobs(excludeDir),
27
35
  });
28
36
  const documents = [];
37
+ const skipped = [];
29
38
  for (const filePath of discovered.sort()) {
30
39
  try {
31
40
  const fileStat = await stat(filePath);
32
- if (!fileStat.isFile() || fileStat.size > maxSizeBytes)
41
+ if (!fileStat.isFile())
42
+ continue;
43
+ if (fileStat.size > maxSizeBytes) {
44
+ skipped.push({
45
+ path: filePath,
46
+ reason: `Exceeds ${Math.round(maxSizeBytes / 1024 / 1024)}MB limit`,
47
+ size: fileStat.size,
48
+ });
33
49
  continue;
50
+ }
34
51
  const ext = path.extname(filePath).toLowerCase().replace('.', '');
35
- if (!isDocumentFormat(ext))
52
+ if (!isDocumentFormat(ext, extensions))
36
53
  continue;
37
54
  const relativePath = normalizeRelativePath(path.relative(resolvedRoot, filePath));
38
55
  if (ignoreMatcher.ignores(relativePath))
@@ -44,10 +61,18 @@ export async function discoverDocuments(rootDir, options) {
44
61
  sizeBytes: fileStat.size,
45
62
  });
46
63
  }
47
- catch {
48
- // Skip files that disappear during discovery
64
+ catch (err) {
65
+ const error = err;
66
+ skipped.push({
67
+ path: filePath,
68
+ reason: error.code === 'EACCES' ? 'Permission denied' : error.message,
69
+ size: 0,
70
+ });
49
71
  }
50
72
  }
51
- return documents;
73
+ return { documents, skipped };
74
+ }
75
+ export async function discoverDocuments(rootDir, options) {
76
+ return (await discoverDocumentSet(rootDir, options)).documents;
52
77
  }
53
78
  //# sourceMappingURL=discover.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../src/doc/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAS/F,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AAI1F,SAAS,gBAAgB,CAAC,GAAW;IACnC,OAAQ,mBAAyC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe,EAAE,OAMxD;IACC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpH,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;IACvE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE;QAC5D,UAAU;QACV,WAAW,EAAE,OAAO,EAAE,WAAW;QACjC,cAAc,EAAE,OAAO,EAAE,cAAc;QACvC,qBAAqB,EAAE,OAAO,EAAE,qBAAqB;KACtD,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,SAAS,mBAAmB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAE1D,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE;QACnC,GAAG,EAAE,YAAY;QACjB,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,KAAK;QACV,mBAAmB,EAAE,KAAK;QAC1B,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC;KACzC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAyB,EAAE,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,GAAG,YAAY;gBAAE,SAAS;YAEjE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC;gBAAE,SAAS;YACrC,MAAM,YAAY,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClF,IAAI,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC;gBAAE,SAAS;YAElD,SAAS,CAAC,IAAI,CAAC;gBACb,QAAQ;gBACR,YAAY;gBACZ,MAAM,EAAE,GAAG;gBACX,SAAS,EAAE,QAAQ,CAAC,IAAI;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6CAA6C;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"discover.js","sourceRoot":"","sources":["../../../src/doc/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAoB/F,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAC;AACjG,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAU,CAAC;AAC9G,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAU,CAAC;AAO1F,SAAS,yBAAyB,CAAC,gBAAyB;IAC1D,OAAO,gBAAgB;QACrB,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,yBAAyB,EAAE,GAAG,wBAAwB,CAAC;QACrF,CAAC,CAAC,CAAC,GAAG,mBAAmB,EAAE,GAAG,yBAAyB,CAAC,CAAC;AAC7D,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAW,EAAE,UAAqC;IAC1E,OAAQ,UAAgC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAe,EAAE,OAAkC;IAC3F,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACpH,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;IACvE,MAAM,aAAa,GAAG,MAAM,mBAAmB,CAAC,YAAY,EAAE;QAC5D,UAAU;QACV,WAAW,EAAE,OAAO,EAAE,WAAW;QACjC,cAAc,EAAE,OAAO,EAAE,cAAc;QACvC,qBAAqB,EAAE,OAAO,EAAE,qBAAqB;KACtD,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,yBAAyB,CAAC,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,SAAS,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAEjD,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE;QACnC,GAAG,EAAE,YAAY;QACjB,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,KAAK;QACV,mBAAmB,EAAE,KAAK;QAC1B,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC;KACzC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAyB,EAAE,CAAC;IAC3C,MAAM,OAAO,GAAqC,EAAE,CAAC;IACrD,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;gBAAE,SAAS;YACjC,IAAI,QAAQ,CAAC,IAAI,GAAG,YAAY,EAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,QAAQ;oBACd,MAAM,EAAE,WAAW,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,GAAG,IAAI,CAAC,UAAU;oBACnE,IAAI,EAAE,QAAQ,CAAC,IAAI;iBACpB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAClE,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,UAAU,CAAC;gBAAE,SAAS;YACjD,MAAM,YAAY,GAAG,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC;YAClF,IAAI,aAAa,CAAC,OAAO,CAAC,YAAY,CAAC;gBAAE,SAAS;YAElD,SAAS,CAAC,IAAI,CAAC;gBACb,QAAQ;gBACR,YAAY;gBACZ,MAAM,EAAE,GAAG;gBACX,SAAS,EAAE,QAAQ,CAAC,IAAI;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,KAAK,GAAG,GAA4B,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO;gBACrE,IAAI,EAAE,CAAC;aACR,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe,EAAE,OAAkC;IACzF,OAAO,CAAC,MAAM,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACjE,CAAC"}
@@ -0,0 +1,6 @@
1
+ export declare const OCC_ABORTED = "OCC_ABORTED";
2
+ export declare class OccAbortError extends Error {
3
+ readonly code = "OCC_ABORTED";
4
+ constructor(message?: string);
5
+ }
6
+ export declare function isOccAbortError(error: unknown): error is OccAbortError;
@@ -0,0 +1,15 @@
1
+ export const OCC_ABORTED = 'OCC_ABORTED';
2
+ export class OccAbortError extends Error {
3
+ code = OCC_ABORTED;
4
+ constructor(message = 'The operation was aborted.') {
5
+ super(message);
6
+ this.name = 'AbortError';
7
+ }
8
+ }
9
+ export function isOccAbortError(error) {
10
+ return error instanceof OccAbortError
11
+ || (error instanceof Error
12
+ && (error.name === 'AbortError'
13
+ || error.code === OCC_ABORTED));
14
+ }
15
+ //# sourceMappingURL=errors.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.js","sourceRoot":"","sources":["../../src/errors.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;AAEzC,MAAM,OAAO,aAAc,SAAQ,KAAK;IAC7B,IAAI,GAAG,WAAW,CAAC;IAE5B,YAAY,OAAO,GAAG,4BAA4B;QAChD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;IAC3B,CAAC;CACF;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,KAAK,YAAY,aAAa;WAChC,CAAC,KAAK,YAAY,KAAK;eACrB,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY;mBACzB,KAA4B,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC;AAChE,CAAC"}
@@ -0,0 +1,12 @@
1
+ export interface OccHealth {
2
+ available: boolean;
3
+ version: string;
4
+ capabilities: {
5
+ code: boolean;
6
+ workspace: boolean;
7
+ documents: boolean;
8
+ tables: boolean;
9
+ health: boolean;
10
+ };
11
+ }
12
+ export declare function health(): Promise<OccHealth>;
@@ -0,0 +1,38 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import path from 'node:path';
3
+ import { fileURLToPath } from 'node:url';
4
+ let cachedVersion;
5
+ async function readVersion() {
6
+ if (cachedVersion)
7
+ return cachedVersion;
8
+ const directory = path.dirname(fileURLToPath(import.meta.url));
9
+ for (const rel of ['..', '../..']) {
10
+ try {
11
+ const raw = await readFile(path.resolve(directory, rel, 'package.json'), 'utf8');
12
+ const parsed = JSON.parse(raw);
13
+ if (parsed.version) {
14
+ cachedVersion = parsed.version;
15
+ return parsed.version;
16
+ }
17
+ }
18
+ catch {
19
+ // Try next package.json location.
20
+ }
21
+ }
22
+ cachedVersion = '0.0.0';
23
+ return cachedVersion;
24
+ }
25
+ export async function health() {
26
+ return {
27
+ available: true,
28
+ version: await readVersion(),
29
+ capabilities: {
30
+ code: true,
31
+ workspace: true,
32
+ documents: true,
33
+ tables: true,
34
+ health: true,
35
+ },
36
+ };
37
+ }
38
+ //# sourceMappingURL=health.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"health.js","sourceRoot":"","sources":["../../src/health.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAczC,IAAI,aAAiC,CAAC;AAEtC,KAAK,UAAU,WAAW;IACxB,IAAI,aAAa;QAAE,OAAO,aAAa,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;YACjF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAyB,CAAC;YACvD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC/B,OAAO,MAAM,CAAC,OAAO,CAAC;YACxB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IACD,aAAa,GAAG,OAAO,CAAC;IACxB,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM;IAC1B,OAAO;QACL,SAAS,EAAE,IAAI;QACf,OAAO,EAAE,MAAM,WAAW,EAAE;QAC5B,YAAY,EAAE;YACZ,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,IAAI;YACf,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,IAAI;SACb;KACF,CAAC;AACJ,CAAC"}
@@ -5,16 +5,23 @@ import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
5
5
  import { discoverCodeFiles } from './code/discover.js';
6
6
  import { previewCodebaseSize } from './code/preview.js';
7
7
  import { buildCodebaseIndexIsolated } from './code/isolated.js';
8
- import { parseSlimIndex, slimifyIndex, validateSlimIndex } from './code/slim.js';
8
+ import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
9
+ import { summarizeModule, toMermaid } from './code/query.js';
10
+ import { buildRepoMap } from './code/map.js';
11
+ import { rankNodes } from './code/rank.js';
9
12
  import { inspectDocument } from './doc/inspect.js';
10
13
  import { discoverDocuments } from './doc/discover.js';
14
+ import { chunkDocument } from './doc/chunk.js';
11
15
  import { inspectWorkbook } from './sheet/inspect.js';
12
16
  import { inspectPresentation } from './slide/inspect.js';
17
+ import { inspectTables } from './table/inspect.js';
13
18
  import { analyzeWorkspace } from './workspace/analyze.js';
14
19
  import { describeWorkspace } from './workspace/describe.js';
15
20
  import { inspectWorkspaceDocumentSet } from './workspace/documents.js';
16
21
  import { prepareWorkspaceContext } from './workspace/prepare.js';
17
- export type { CodebaseIndex, CodeNode, CodeEdge, CodeNodeType, CodeEdgeType, IndexTruncation } from './code/types.js';
22
+ import { bundleWorkspace } from './workspace/bundle.js';
23
+ import { health } from './health.js';
24
+ export type { CodebaseIndex, CodeNode, CodeEdge, CodeNodeType, CodeEdgeType, IndexTruncation, NormalizedSymbolKind } from './code/types.js';
18
25
  export type { CodeQuerySession, CodeSessionChunkOptions } from './code/session.js';
19
26
  export type { CodeIndexStore, CodeIndexStoreOptions, StoreStrategy, StoreMetadata, StoreSessionOptions } from './code/store.js';
20
27
  export type { CodeChunk, CodeChunkOptions } from './code/chunk.js';
@@ -22,14 +29,26 @@ export type { BuildCodebaseOptions, CodeIndexBudget, BudgetExceededBehavior } fr
22
29
  export type { BuildCodebaseIndexIsolatedOptions } from './code/isolated.js';
23
30
  export type { CodebaseIndexSlim, SlimParsedFile } from './code/slim.js';
24
31
  export type { CodebaseSizePreview, PreviewCodebaseSizeOptions } from './code/preview.js';
32
+ export type { ModuleSummary, ModuleSummaryOptions, MermaidKind, MermaidOptions } from './code/query.js';
33
+ export type { RepoMapOptions, RepoMapResult, RepoMapEntry, RepoMapSymbol, RepoMapFormat, RepoMapMode } from './code/map.js';
34
+ export type { RankOptions, RankResult, RankedNode, RankedFile, RankEdgeWeights } from './code/rank.js';
35
+ export type { DocumentChunk, DocumentChunkOptions } from './doc/chunk.js';
25
36
  export type { OnProgress, ProgressEvent, ProgressPhase } from './progress-event.js';
26
37
  export type { WorkspacePrepareOptions, WorkspacePreparedContext, WorkspacePreparationProgressEvent, OnPrepareProgress } from './workspace/prepare-types.js';
27
38
  export type { AnalyzeWorkspaceOptions } from './workspace/analyze.js';
28
39
  export type { DescribeWorkspaceOptions } from './workspace/describe.js';
29
40
  export type { InspectWorkspaceDocumentSetOptions } from './workspace/documents.js';
41
+ export type { WorkspaceBundle, WorkspaceBundleOptions, WorkspaceOutlineNode, WorkspaceCodeDocumentReference } from './workspace/bundle.js';
30
42
  export type { WorkspaceDescription, WorkspaceRecommendedCall } from './workspace/types.js';
31
- export { buildCodebaseIndex, buildCodebaseIndexIsolated, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, validateSlimIndex, };
43
+ export type { OccHealth } from './health.js';
44
+ export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
32
45
  export { CodeIndexBudgetExceededError } from './code/build.js';
46
+ export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
47
+ export { health } from './health.js';
48
+ export { inspectTables } from './table/inspect.js';
49
+ export { chunkDocument } from './doc/chunk.js';
50
+ export { bundleWorkspace } from './workspace/bundle.js';
51
+ export { toNormalizedSymbolKind } from './code/types.js';
33
52
  export interface OccCodeNamespace {
34
53
  buildIndex: typeof buildCodebaseIndex;
35
54
  buildIndexIsolated: typeof buildCodebaseIndexIsolated;
@@ -41,8 +60,13 @@ export interface OccCodeNamespace {
41
60
  discover: typeof discoverCodeFiles;
42
61
  chunk: typeof chunkCodebase;
43
62
  chunkFromIndex: typeof chunkFromIndex;
63
+ summarizeModule: typeof summarizeModule;
64
+ toMermaid: typeof toMermaid;
65
+ map: typeof buildRepoMap;
66
+ rank: typeof rankNodes;
44
67
  slimifyIndex: typeof slimifyIndex;
45
68
  parseSlimIndex: typeof parseSlimIndex;
69
+ tryParseSlimIndex: typeof tryParseSlimIndex;
46
70
  validateSlimIndex: typeof validateSlimIndex;
47
71
  }
48
72
  export interface OccWorkspaceNamespace {
@@ -50,10 +74,12 @@ export interface OccWorkspaceNamespace {
50
74
  describe: typeof describeWorkspace;
51
75
  inspectDocuments: typeof inspectWorkspaceDocumentSet;
52
76
  prepare: typeof prepareWorkspaceContext;
77
+ bundle: typeof bundleWorkspace;
53
78
  }
54
79
  export interface OccDocNamespace {
55
80
  inspect: typeof inspectDocument;
56
81
  discover: typeof discoverDocuments;
82
+ chunk: typeof chunkDocument;
57
83
  }
58
84
  export interface OccSheetNamespace {
59
85
  inspect: typeof inspectWorkbook;
@@ -61,11 +87,16 @@ export interface OccSheetNamespace {
61
87
  export interface OccSlideNamespace {
62
88
  inspect: typeof inspectPresentation;
63
89
  }
90
+ export interface OccTableNamespace {
91
+ inspect: typeof inspectTables;
92
+ }
64
93
  export interface OccNamespace {
65
94
  code: OccCodeNamespace;
66
95
  workspace: OccWorkspaceNamespace;
67
96
  doc: OccDocNamespace;
68
97
  sheet: OccSheetNamespace;
69
98
  slide: OccSlideNamespace;
99
+ table: OccTableNamespace;
100
+ health: typeof health;
70
101
  }
71
102
  export declare function createOcc(): OccNamespace;
package/dist/src/index.js CHANGED
@@ -5,17 +5,30 @@ import { chunkCodebase, chunkFromIndex } from './code/chunk.js';
5
5
  import { discoverCodeFiles } from './code/discover.js';
6
6
  import { previewCodebaseSize } from './code/preview.js';
7
7
  import { buildCodebaseIndexIsolated } from './code/isolated.js';
8
- import { parseSlimIndex, slimifyIndex, validateSlimIndex } from './code/slim.js';
8
+ import { parseSlimIndex, slimifyIndex, tryParseSlimIndex, validateSlimIndex } from './code/slim.js';
9
+ import { summarizeModule, toMermaid } from './code/query.js';
10
+ import { buildRepoMap } from './code/map.js';
11
+ import { rankNodes } from './code/rank.js';
9
12
  import { inspectDocument } from './doc/inspect.js';
10
13
  import { discoverDocuments } from './doc/discover.js';
14
+ import { chunkDocument } from './doc/chunk.js';
11
15
  import { inspectWorkbook } from './sheet/inspect.js';
12
16
  import { inspectPresentation } from './slide/inspect.js';
17
+ import { inspectTables } from './table/inspect.js';
13
18
  import { analyzeWorkspace } from './workspace/analyze.js';
14
19
  import { describeWorkspace } from './workspace/describe.js';
15
20
  import { inspectWorkspaceDocumentSet } from './workspace/documents.js';
16
21
  import { prepareWorkspaceContext } from './workspace/prepare.js';
17
- export { buildCodebaseIndex, buildCodebaseIndexIsolated, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, validateSlimIndex, };
22
+ import { bundleWorkspace } from './workspace/bundle.js';
23
+ import { health } from './health.js';
24
+ export { buildCodebaseIndex, buildCodebaseIndexIsolated, buildRepoMap, rankNodes, chunkCodebase, chunkFromIndex, createCodeQuerySession, createCodeQuerySessionFromIndex, discoverCodeFiles, openChunkCodeIndexStore, openChunkCodeIndexStore as openChunkStore, openCodeIndexStore, parseSlimIndex, previewCodebaseSize, slimifyIndex, summarizeModule, toMermaid, tryParseSlimIndex, validateSlimIndex, };
18
25
  export { CodeIndexBudgetExceededError } from './code/build.js';
26
+ export { OccAbortError, OCC_ABORTED, isOccAbortError } from './errors.js';
27
+ export { health } from './health.js';
28
+ export { inspectTables } from './table/inspect.js';
29
+ export { chunkDocument } from './doc/chunk.js';
30
+ export { bundleWorkspace } from './workspace/bundle.js';
31
+ export { toNormalizedSymbolKind } from './code/types.js';
19
32
  export function createOcc() {
20
33
  return {
21
34
  code: {
@@ -29,8 +42,13 @@ export function createOcc() {
29
42
  discover: discoverCodeFiles,
30
43
  chunk: chunkCodebase,
31
44
  chunkFromIndex: chunkFromIndex,
45
+ summarizeModule,
46
+ toMermaid,
47
+ map: buildRepoMap,
48
+ rank: rankNodes,
32
49
  slimifyIndex,
33
50
  parseSlimIndex,
51
+ tryParseSlimIndex,
34
52
  validateSlimIndex,
35
53
  },
36
54
  workspace: {
@@ -38,10 +56,12 @@ export function createOcc() {
38
56
  describe: describeWorkspace,
39
57
  inspectDocuments: inspectWorkspaceDocumentSet,
40
58
  prepare: prepareWorkspaceContext,
59
+ bundle: bundleWorkspace,
41
60
  },
42
61
  doc: {
43
62
  inspect: inspectDocument,
44
63
  discover: discoverDocuments,
64
+ chunk: chunkDocument,
45
65
  },
46
66
  sheet: {
47
67
  inspect: inspectWorkbook,
@@ -49,6 +69,10 @@ export function createOcc() {
49
69
  slide: {
50
70
  inspect: inspectPresentation,
51
71
  },
72
+ table: {
73
+ inspect: inspectTables,
74
+ },
75
+ health,
52
76
  };
53
77
  }
54
78
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,+BAA+B,GAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAkBjE,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,aAAa,EACb,cAAc,EACd,sBAAsB,EACtB,+BAA+B,EAC/B,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,IAAI,cAAc,EACzC,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,iBAAiB,GAClB,CAAC;AACF,OAAO,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AA8C/D,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,IAAI,EAAE;YACJ,UAAU,EAAE,kBAAkB;YAC9B,kBAAkB,EAAE,0BAA0B;YAC9C,aAAa,EAAE,sBAAsB;YACrC,sBAAsB,EAAE,+BAA+B;YACvD,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,uBAAuB;YACvC,WAAW,EAAE,mBAAmB;YAChC,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;YACpB,cAAc,EAAE,cAAc;YAC9B,YAAY;YACZ,cAAc;YACd,iBAAiB;SAClB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,gBAAgB;YACzB,QAAQ,EAAE,iBAAiB;YAC3B,gBAAgB,EAAE,2BAA2B;YAC7C,OAAO,EAAE,uBAAuB;SACjC;QACD,GAAG,EAAE;YACH,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,iBAAiB;SAC5B;QACD,KAAK,EAAE;YACL,OAAO,EAAE,eAAe;SACzB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,mBAAmB;SAC7B;KACF,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,+BAA+B,GAChC,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC9E,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAwBrC,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,YAAY,EACZ,SAAS,EACT,aAAa,EACb,cAAc,EACd,sBAAsB,EACtB,+BAA+B,EAC/B,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,IAAI,cAAc,EACzC,kBAAkB,EAClB,cAAc,EACd,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,SAAS,EACT,iBAAiB,EACjB,iBAAiB,GAClB,CAAC;AACF,OAAO,EAAE,4BAA4B,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AA2DzD,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,IAAI,EAAE;YACJ,UAAU,EAAE,kBAAkB;YAC9B,kBAAkB,EAAE,0BAA0B;YAC9C,aAAa,EAAE,sBAAsB;YACrC,sBAAsB,EAAE,+BAA+B;YACvD,SAAS,EAAE,kBAAkB;YAC7B,cAAc,EAAE,uBAAuB;YACvC,WAAW,EAAE,mBAAmB;YAChC,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;YACpB,cAAc,EAAE,cAAc;YAC9B,eAAe;YACf,SAAS;YACT,GAAG,EAAE,YAAY;YACjB,IAAI,EAAE,SAAS;YACf,YAAY;YACZ,cAAc;YACd,iBAAiB;YACjB,iBAAiB;SAClB;QACD,SAAS,EAAE;YACT,OAAO,EAAE,gBAAgB;YACzB,QAAQ,EAAE,iBAAiB;YAC3B,gBAAgB,EAAE,2BAA2B;YAC7C,OAAO,EAAE,uBAAuB;YAChC,MAAM,EAAE,eAAe;SACxB;QACD,GAAG,EAAE;YACH,OAAO,EAAE,eAAe;YACxB,QAAQ,EAAE,iBAAiB;YAC3B,KAAK,EAAE,aAAa;SACrB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,eAAe;SACzB;QACD,KAAK,EAAE;YACL,OAAO,EAAE,mBAAmB;SAC7B;QACD,KAAK,EAAE;YACL,OAAO,EAAE,aAAa;SACvB;QACD,MAAM;KACP,CAAC;AACJ,CAAC"}
@@ -8,6 +8,20 @@ import { z } from 'zod';
8
8
  import { extractOfficeText, getExtension } from '../utils.js';
9
9
  const turndown = new TurndownService({ headingStyle: 'atx' });
10
10
  const PdfTextContentSchema = z.object({ items: z.array(z.object({ str: z.string() }).passthrough()) });
11
+ const RAW_TEXT_EXTENSIONS = new Set([
12
+ 'md',
13
+ 'markdown',
14
+ 'mdx',
15
+ 'txt',
16
+ 'rst',
17
+ 'adoc',
18
+ 'asciidoc',
19
+ 'yaml',
20
+ 'yml',
21
+ 'json',
22
+ 'jsonc',
23
+ 'toml',
24
+ ]);
11
25
  /** Convert a DOCX file to markdown via mammoth → HTML → turndown */
12
26
  async function docxToMarkdown(filePath) {
13
27
  const result = await mammoth.convertToHtml({ path: filePath });
@@ -154,6 +168,9 @@ async function xlsxToMarkdown(filePath, maxRows = 20) {
154
168
  /** Convert a document to markdown. Returns null for unsupported formats (ods). */
155
169
  export async function documentToMarkdown(filePath) {
156
170
  const ext = getExtension(filePath);
171
+ if (RAW_TEXT_EXTENSIONS.has(ext)) {
172
+ return readFile(filePath, 'utf8');
173
+ }
157
174
  switch (ext) {
158
175
  case 'docx': return docxToMarkdown(filePath);
159
176
  case 'pdf': return pdfToMarkdown(filePath);
@@ -1 +1 @@
1
- {"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/markdown/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,GAAG,MAAM,WAAW,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAC9D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;AAEvG,oEAAoE;AACpE,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,sDAAsD;AACtD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;IAChC,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;QACnC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC;YACrJ,OAAO;QACT,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC;IAEF,IAAI,IAAwC,CAAC;IAC7C,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE;YACvB,UAAU,EAAE,KAAK,EAAE,QAAuE,EAAE,EAAE;gBAC5F,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;gBAChF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACxD,OAAO,SAAS,QAAQ,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,CAAC;SACF,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,GAAG,WAAW,CAAC;IAC5B,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC;AACnB,CAAC;AAED,oDAAoD;AACpD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE1C,uCAAuC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;SACtC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACzD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,OAAO,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IAEL,iCAAiC;IACjC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;IAErC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,gBAAgB,QAAQ,EAAE,CAAC;IACpC,CAAC;IAED,oCAAoC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,sEAAsE;AACtE,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAE3B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE7C,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,OAAO,gBAAgB,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,OAAO,GAAG,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,IAAI;QACf,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC;YAAE,SAAS;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE5C,KAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,kBAAkB;QAClB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrE,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;QAED,uBAAuB;QACvB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;YACjE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,GAAG,OAAO,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,GAAG,CAAC,GAAG,OAAO,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACvD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEnC,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK;YACR,OAAO,IAAI,CAAC;QACd;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"convert.js","sourceRoot":"","sources":["../../../src/markdown/convert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,GAAG,MAAM,WAAW,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE9D,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAC9D,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;AACvG,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,IAAI;IACJ,UAAU;IACV,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,UAAU;IACV,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC,CAAC;AAEH,oEAAoE;AACpE,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,sDAAsD;AACtD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAExC,2BAA2B;IAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC;IAChC,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE;QACnC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC;YACrJ,OAAO;QACT,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC;IAEF,IAAI,IAAwC,CAAC;IAC7C,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,GAAG,CAAC,MAAM,EAAE;YACvB,UAAU,EAAE,KAAK,EAAE,QAAuE,EAAE,EAAE;gBAC5F,MAAM,WAAW,GAAG,oBAAoB,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;gBAChF,MAAM,OAAO,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACxD,OAAO,SAAS,QAAQ,CAAC,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAClE,CAAC;SACF,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,GAAG,WAAW,CAAC;IAC5B,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC;AACnB,CAAC;AAED,oDAAoD;AACpD,KAAK,UAAU,cAAc,CAAC,QAAgB;IAC5C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAE1C,uCAAuC;IACvC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;SACtC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACzD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7D,OAAO,IAAI,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IAEL,iCAAiC;IACjC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;IAErC,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,gBAAgB,QAAQ,EAAE,CAAC;IACpC,CAAC;IAED,oCAAoC;IACpC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,sEAAsE;AACtE,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yDAAyD;AACzD,KAAK,UAAU,aAAa,CAAC,QAAgB;IAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAE3B,MAAM,MAAM,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC/D,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE7C,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,OAAO,gBAAgB,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,GAAG,aAAa,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,SAAS,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,uEAAuE;AACvE,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,OAAO,GAAG,EAAE;IAC1D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QACjC,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,IAAI;QACf,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC;YAAE,SAAS;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAE5C,KAAK,CAAC,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QAED,kBAAkB;QAClB,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;YACrE,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAC9D,CAAC;QAED,uBAAuB;QACvB,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE1D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,EAAE,EAAE,CAAC;YACjE,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC/D,MAAM,GAAG,GAAG,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,GAAG,OAAO,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,OAAO,SAAS,GAAG,CAAC,GAAG,OAAO,aAAa,CAAC,CAAC;QAC1D,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,QAAgB;IACvD,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;IAEnC,IAAI,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,KAAK,CAAC,CAAC,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,CAAC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,KAAK;YACR,OAAO,IAAI,CAAC;QACd;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
@@ -1,8 +1,14 @@
1
- export type ProgressPhase = 'start' | 'discover' | 'parse' | 'index' | 'graph' | 'serialize' | 'deserialize' | 'cache-check' | 'cache-load' | 'cache-save' | 'chunk' | 'inspect' | 'references' | 'cross-references' | 'discover-documents' | 'parse-documents' | 'structure' | 'code' | 'complete';
1
+ export type ProgressPhase = 'start' | 'bundle' | 'description' | 'stats' | 'code-preview' | 'discover' | 'parse' | 'index' | 'graph' | 'serialize' | 'deserialize' | 'cache-check' | 'cache-load' | 'cache-save' | 'chunk' | 'inspect' | 'references' | 'cross-references' | 'document-chunk' | 'discover-documents' | 'parse-documents' | 'structure' | 'outline' | 'code' | 'complete';
2
2
  export interface ProgressEvent {
3
3
  phase: ProgressPhase;
4
4
  total: number;
5
5
  completed: number;
6
6
  detail?: string;
7
+ scope?: string;
8
+ currentPath?: string;
9
+ bytesProcessed?: number;
10
+ totalBytes?: number;
11
+ startedAt?: number;
12
+ elapsedMs?: number;
7
13
  }
8
14
  export type OnProgress = (event: ProgressEvent) => void;
@@ -1,7 +1,14 @@
1
1
  /**
2
2
  * Language-aware token estimation — tokenx-inspired approach.
3
3
  * Replaces the naive `chars / 4` heuristic with segment-level analysis.
4
+ *
5
+ * The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
6
+ * zero-dependency default. For exact, model-specific counts, use the
7
+ * pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
8
+ * loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
9
+ * only when requested.
4
10
  */
11
+ import { z } from 'zod';
5
12
  export interface LanguageConfig {
6
13
  pattern: RegExp;
7
14
  averageCharsPerToken: number;
@@ -14,3 +21,27 @@ export interface TokenEstimationOptions {
14
21
  export declare function estimateTokenCount(text?: string | number, options?: TokenEstimationOptions): number;
15
22
  /** Slice text by token positions (like Array.slice but for tokens) */
16
23
  export declare function sliceByTokens(text: string, start?: number, end?: number, options?: TokenEstimationOptions): string;
24
+ export declare const TokenizerNameSchema: z.ZodEnum<{
25
+ heuristic: "heuristic";
26
+ o200k_base: "o200k_base";
27
+ cl100k_base: "cl100k_base";
28
+ }>;
29
+ export type TokenizerName = z.infer<typeof TokenizerNameSchema>;
30
+ export interface Tokenizer {
31
+ /** Identifier of the active encoding (`heuristic` for the language-aware estimate). */
32
+ readonly name: TokenizerName;
33
+ /** Token count for `text`. A number input is treated as a character count (heuristic only). */
34
+ count(text: string | number): number;
35
+ /** Slice `text` by token positions, like `Array.prototype.slice` but token-aware. */
36
+ slice(text: string, start?: number, end?: number): string;
37
+ }
38
+ /**
39
+ * Build a tokenizer for the given encoding. `heuristic` resolves synchronously
40
+ * with no dependency; BPE encodings lazily load their tables once (cached).
41
+ */
42
+ export declare function createTokenizer(name: TokenizerName): Promise<Tokenizer>;
43
+ /**
44
+ * Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
45
+ * env var, defaulting to `heuristic`. Throws on an unrecognized value.
46
+ */
47
+ export declare function resolveTokenizerName(explicit?: string, env?: string | undefined): TokenizerName;
@@ -1,7 +1,14 @@
1
1
  /**
2
2
  * Language-aware token estimation — tokenx-inspired approach.
3
3
  * Replaces the naive `chars / 4` heuristic with segment-level analysis.
4
+ *
5
+ * The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
6
+ * zero-dependency default. For exact, model-specific counts, use the
7
+ * pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
8
+ * loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
9
+ * only when requested.
4
10
  */
11
+ import { z } from 'zod';
5
12
  const PATTERNS = {
6
13
  whitespace: /^\s+$/,
7
14
  cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
@@ -104,4 +111,85 @@ export function sliceByTokens(text, start = 0, end, options = {}) {
104
111
  }
105
112
  return parts.join('');
106
113
  }
114
+ // ── Pluggable tokenizers ──────────────────────────────────────────────────
115
+ //
116
+ // `Tokenizer` lets callers swap the heuristic estimator for an exact BPE
117
+ // tokenizer without changing call sites: every consumer takes a
118
+ // `(text) => number` counter (or a `Tokenizer`) and threads it down. There is
119
+ // deliberately NO mutable module-level "current tokenizer" — the codebase
120
+ // keeps zero global state, so the selected tokenizer is always passed
121
+ // explicitly (resolved once at an entry point via `resolveTokenizerName` +
122
+ // `createTokenizer`).
123
+ export const TokenizerNameSchema = z.enum(['heuristic', 'o200k_base', 'cl100k_base']);
124
+ class HeuristicTokenizer {
125
+ name = 'heuristic';
126
+ count(text) {
127
+ return estimateTokenCount(text);
128
+ }
129
+ slice(text, start = 0, end) {
130
+ return sliceByTokens(text, start, end);
131
+ }
132
+ }
133
+ const encodingCache = new Map();
134
+ async function loadEncoding(name) {
135
+ const cached = encodingCache.get(name);
136
+ if (cached)
137
+ return cached;
138
+ const mod = name === 'o200k_base'
139
+ ? await import('gpt-tokenizer/encoding/o200k_base')
140
+ : await import('gpt-tokenizer/encoding/cl100k_base');
141
+ encodingCache.set(name, mod);
142
+ return mod;
143
+ }
144
+ class BpeTokenizer {
145
+ name;
146
+ encoding;
147
+ constructor(name, encoding) {
148
+ this.name = name;
149
+ this.encoding = encoding;
150
+ }
151
+ count(text) {
152
+ // A bare character count has no text to encode; fall back to the heuristic.
153
+ if (typeof text === 'number')
154
+ return estimateTokenCount(text);
155
+ if (!text)
156
+ return 0;
157
+ return this.encoding.encode(text).length;
158
+ }
159
+ slice(text, start = 0, end) {
160
+ if (!text)
161
+ return '';
162
+ const tokens = this.encoding.encode(text);
163
+ const total = tokens.length;
164
+ const from = start < 0 ? Math.max(0, total + start) : Math.min(start, total);
165
+ const to = end === undefined ? total : end < 0 ? Math.max(0, total + end) : Math.min(end, total);
166
+ if (from >= to)
167
+ return '';
168
+ return this.encoding.decode(tokens.slice(from, to));
169
+ }
170
+ }
171
+ /**
172
+ * Build a tokenizer for the given encoding. `heuristic` resolves synchronously
173
+ * with no dependency; BPE encodings lazily load their tables once (cached).
174
+ */
175
+ export async function createTokenizer(name) {
176
+ if (name === 'heuristic')
177
+ return new HeuristicTokenizer();
178
+ const encoding = await loadEncoding(name);
179
+ return new BpeTokenizer(name, encoding);
180
+ }
181
+ /**
182
+ * Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
183
+ * env var, defaulting to `heuristic`. Throws on an unrecognized value.
184
+ */
185
+ export function resolveTokenizerName(explicit, env = process.env.OCC_TOKENIZER) {
186
+ const candidate = explicit ?? env;
187
+ if (candidate == null || candidate === '')
188
+ return 'heuristic';
189
+ const result = TokenizerNameSchema.safeParse(candidate);
190
+ if (!result.success) {
191
+ throw new Error(`Invalid tokenizer "${candidate}". Expected one of: ${TokenizerNameSchema.options.join(', ')}`);
192
+ }
193
+ return result.data;
194
+ }
107
195
  //# sourceMappingURL=tokens.js.map