@cesarandreslopez/occ 0.8.8 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/src/cli.d.ts +2 -1
- package/dist/src/cli.js +9 -6
- package/dist/src/cli.js.map +1 -1
- package/dist/src/code/command.js +65 -1
- package/dist/src/code/command.js.map +1 -1
- package/dist/src/code/focus.d.ts +31 -0
- package/dist/src/code/focus.js +192 -0
- package/dist/src/code/focus.js.map +1 -0
- package/dist/src/code/map.d.ts +173 -0
- package/dist/src/code/map.js +344 -0
- package/dist/src/code/map.js.map +1 -0
- package/dist/src/code/output.d.ts +5 -0
- package/dist/src/code/output.js +114 -0
- package/dist/src/code/output.js.map +1 -1
- package/dist/src/code/rank.d.ts +86 -0
- package/dist/src/code/rank.js +208 -0
- package/dist/src/code/rank.js.map +1 -0
- package/dist/src/index.d.ts +7 -1
- package/dist/src/index.js +5 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/tokens.d.ts +31 -0
- package/dist/src/tokens.js +88 -0
- package/dist/src/tokens.js.map +1 -1
- package/package.json +2 -1
package/dist/src/tokens.d.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Language-aware token estimation — tokenx-inspired approach.
|
|
3
3
|
* Replaces the naive `chars / 4` heuristic with segment-level analysis.
|
|
4
|
+
*
|
|
5
|
+
* The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
|
|
6
|
+
* zero-dependency default. For exact, model-specific counts, use the
|
|
7
|
+
* pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
|
|
8
|
+
* loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
|
|
9
|
+
* only when requested.
|
|
4
10
|
*/
|
|
11
|
+
import { z } from 'zod';
|
|
5
12
|
export interface LanguageConfig {
|
|
6
13
|
pattern: RegExp;
|
|
7
14
|
averageCharsPerToken: number;
|
|
@@ -14,3 +21,27 @@ export interface TokenEstimationOptions {
|
|
|
14
21
|
export declare function estimateTokenCount(text?: string | number, options?: TokenEstimationOptions): number;
|
|
15
22
|
/** Slice text by token positions (like Array.slice but for tokens) */
|
|
16
23
|
export declare function sliceByTokens(text: string, start?: number, end?: number, options?: TokenEstimationOptions): string;
|
|
24
|
+
export declare const TokenizerNameSchema: z.ZodEnum<{
|
|
25
|
+
heuristic: "heuristic";
|
|
26
|
+
o200k_base: "o200k_base";
|
|
27
|
+
cl100k_base: "cl100k_base";
|
|
28
|
+
}>;
|
|
29
|
+
export type TokenizerName = z.infer<typeof TokenizerNameSchema>;
|
|
30
|
+
export interface Tokenizer {
|
|
31
|
+
/** Identifier of the active encoding (`heuristic` for the language-aware estimate). */
|
|
32
|
+
readonly name: TokenizerName;
|
|
33
|
+
/** Token count for `text`. A number input is treated as a character count (heuristic only). */
|
|
34
|
+
count(text: string | number): number;
|
|
35
|
+
/** Slice `text` by token positions, like `Array.prototype.slice` but token-aware. */
|
|
36
|
+
slice(text: string, start?: number, end?: number): string;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Build a tokenizer for the given encoding. `heuristic` resolves synchronously
|
|
40
|
+
* with no dependency; BPE encodings lazily load their tables once (cached).
|
|
41
|
+
*/
|
|
42
|
+
export declare function createTokenizer(name: TokenizerName): Promise<Tokenizer>;
|
|
43
|
+
/**
|
|
44
|
+
* Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
|
|
45
|
+
* env var, defaulting to `heuristic`. Throws on an unrecognized value.
|
|
46
|
+
*/
|
|
47
|
+
export declare function resolveTokenizerName(explicit?: string, env?: string | undefined): TokenizerName;
|
package/dist/src/tokens.js
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Language-aware token estimation — tokenx-inspired approach.
|
|
3
3
|
* Replaces the naive `chars / 4` heuristic with segment-level analysis.
|
|
4
|
+
*
|
|
5
|
+
* The heuristic estimator (`estimateTokenCount`/`sliceByTokens`) is the
|
|
6
|
+
* zero-dependency default. For exact, model-specific counts, use the
|
|
7
|
+
* pluggable {@link Tokenizer} abstraction (`createTokenizer`), which lazily
|
|
8
|
+
* loads OpenAI BPE encodings (`o200k_base`, `cl100k_base`) via `gpt-tokenizer`
|
|
9
|
+
* only when requested.
|
|
4
10
|
*/
|
|
11
|
+
import { z } from 'zod';
|
|
5
12
|
const PATTERNS = {
|
|
6
13
|
whitespace: /^\s+$/,
|
|
7
14
|
cjk: /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/,
|
|
@@ -104,4 +111,85 @@ export function sliceByTokens(text, start = 0, end, options = {}) {
|
|
|
104
111
|
}
|
|
105
112
|
return parts.join('');
|
|
106
113
|
}
|
|
114
|
+
// ── Pluggable tokenizers ──────────────────────────────────────────────────
|
|
115
|
+
//
|
|
116
|
+
// `Tokenizer` lets callers swap the heuristic estimator for an exact BPE
|
|
117
|
+
// tokenizer without changing call sites: every consumer takes a
|
|
118
|
+
// `(text) => number` counter (or a `Tokenizer`) and threads it down. There is
|
|
119
|
+
// deliberately NO mutable module-level "current tokenizer" — the codebase
|
|
120
|
+
// keeps zero global state, so the selected tokenizer is always passed
|
|
121
|
+
// explicitly (resolved once at an entry point via `resolveTokenizerName` +
|
|
122
|
+
// `createTokenizer`).
|
|
123
|
+
export const TokenizerNameSchema = z.enum(['heuristic', 'o200k_base', 'cl100k_base']);
|
|
124
|
+
class HeuristicTokenizer {
|
|
125
|
+
name = 'heuristic';
|
|
126
|
+
count(text) {
|
|
127
|
+
return estimateTokenCount(text);
|
|
128
|
+
}
|
|
129
|
+
slice(text, start = 0, end) {
|
|
130
|
+
return sliceByTokens(text, start, end);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const encodingCache = new Map();
|
|
134
|
+
async function loadEncoding(name) {
|
|
135
|
+
const cached = encodingCache.get(name);
|
|
136
|
+
if (cached)
|
|
137
|
+
return cached;
|
|
138
|
+
const mod = name === 'o200k_base'
|
|
139
|
+
? await import('gpt-tokenizer/encoding/o200k_base')
|
|
140
|
+
: await import('gpt-tokenizer/encoding/cl100k_base');
|
|
141
|
+
encodingCache.set(name, mod);
|
|
142
|
+
return mod;
|
|
143
|
+
}
|
|
144
|
+
class BpeTokenizer {
|
|
145
|
+
name;
|
|
146
|
+
encoding;
|
|
147
|
+
constructor(name, encoding) {
|
|
148
|
+
this.name = name;
|
|
149
|
+
this.encoding = encoding;
|
|
150
|
+
}
|
|
151
|
+
count(text) {
|
|
152
|
+
// A bare character count has no text to encode; fall back to the heuristic.
|
|
153
|
+
if (typeof text === 'number')
|
|
154
|
+
return estimateTokenCount(text);
|
|
155
|
+
if (!text)
|
|
156
|
+
return 0;
|
|
157
|
+
return this.encoding.encode(text).length;
|
|
158
|
+
}
|
|
159
|
+
slice(text, start = 0, end) {
|
|
160
|
+
if (!text)
|
|
161
|
+
return '';
|
|
162
|
+
const tokens = this.encoding.encode(text);
|
|
163
|
+
const total = tokens.length;
|
|
164
|
+
const from = start < 0 ? Math.max(0, total + start) : Math.min(start, total);
|
|
165
|
+
const to = end === undefined ? total : end < 0 ? Math.max(0, total + end) : Math.min(end, total);
|
|
166
|
+
if (from >= to)
|
|
167
|
+
return '';
|
|
168
|
+
return this.encoding.decode(tokens.slice(from, to));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Build a tokenizer for the given encoding. `heuristic` resolves synchronously
|
|
173
|
+
* with no dependency; BPE encodings lazily load their tables once (cached).
|
|
174
|
+
*/
|
|
175
|
+
export async function createTokenizer(name) {
|
|
176
|
+
if (name === 'heuristic')
|
|
177
|
+
return new HeuristicTokenizer();
|
|
178
|
+
const encoding = await loadEncoding(name);
|
|
179
|
+
return new BpeTokenizer(name, encoding);
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Resolve a tokenizer name from an explicit option, then the `OCC_TOKENIZER`
|
|
183
|
+
* env var, defaulting to `heuristic`. Throws on an unrecognized value.
|
|
184
|
+
*/
|
|
185
|
+
export function resolveTokenizerName(explicit, env = process.env.OCC_TOKENIZER) {
|
|
186
|
+
const candidate = explicit ?? env;
|
|
187
|
+
if (candidate == null || candidate === '')
|
|
188
|
+
return 'heuristic';
|
|
189
|
+
const result = TokenizerNameSchema.safeParse(candidate);
|
|
190
|
+
if (!result.success) {
|
|
191
|
+
throw new Error(`Invalid tokenizer "${candidate}". Expected one of: ${TokenizerNameSchema.options.join(', ')}`);
|
|
192
|
+
}
|
|
193
|
+
return result.data;
|
|
194
|
+
}
|
|
107
195
|
//# sourceMappingURL=tokens.js.map
|
package/dist/src/tokens.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/tokens.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,MAAM,QAAQ,GAAG;IACf,UAAU,EAAE,OAAO;IACnB,GAAG,EAAE,0LAA0L;IAC/L,OAAO,EAAE,mBAAmB;IAC5B,WAAW,EAAE,oCAAoC;IACjD,YAAY,EAAE,uDAAuD;CAC7D,CAAC;AAEX,MAAM,mBAAmB,GAAG,IAAI,MAAM,CAAC,SAAS,QAAQ,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC;AACjF,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEhC,MAAM,wBAAwB,GAAqB;IACjD,EAAE,OAAO,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC,EAAE;IAChD,EAAE,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,CAAC,EAAE;IAChE,EAAE,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,GAAG,EAAE;CAClE,CAAC;AAEF,SAAS,gCAAgC,CAAC,OAAe,EAAE,OAAyB;IAClF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,OAAO,MAAM,CAAC,oBAAoB,CAAC;QACrC,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACjC,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAAe,EACf,eAAiC,EACjC,oBAA4B;IAE5B,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,CAAC,CAAC;IAChD,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAClE,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,MAAM,IAAI,qBAAqB;QAAE,OAAO,CAAC,CAAC;IACtD,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtG,MAAM,aAAa,GAAG,gCAAgC,CAAC,OAAO,EAAE,eAAe,CAAC,IAAI,oBAAoB,CAAC;IACzG,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC;AACnD,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,kBAAkB,CAAC,IAAsB,EAAE,UAAkC,EAAE;IAC7F,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,CAAC,CAAC;IAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,mEAAmE;QACnE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,uBAAuB,CAAC,CAAC,CAAC,CAAC;IAClG,CAAC;IACD,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,MAAM,EACJ,oBAAoB,GAAG,uBAAuB,EAC9C,eAAe,GAAG,wBAAwB,GAC3C,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,UAAU,IAAI,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,KAAK,GAAG,CAAC,EACT,GAAY,EACZ,UAAkC,EAAE;IAEpC,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,EAAE,oBAAoB,GAAG,uBAAuB,EAAE,eAAe,GAAG,wBAAwB,EAAE,GAAG,OAAO,CAAC;IAE/G,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,WAAW,GAAG,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,eAAe,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1F,MAAM,aAAa,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEpG,IAAI,eAAe,IAAI,aAAa;QAAE,OAAO,EAAE,CAAC;IAEhD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACjE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,eAAe,IAAI,aAAa;YAAE,MAAM;QAE5C,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;QACzF,MAAM,UAAU,GAAG,eAAe,GAAG,UAAU,CAAC;QAEhD,IAAI,UAAU,GAAG,eAAe,IAAI,eAAe,GAAG,aAAa,EAAE,CAAC;YACpE,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACrB,IAAI,eAAe,IAAI,eAAe,IAAI,eAAe,GAAG,aAAa,EAAE,CAAC;oBAC1E,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,eAAe,CAAC,CAAC;gBACpE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,GAAG,eAAe,CAAC,CAAC;gBACzE,IAAI,YAAY,KAAK,CAAC,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;oBACpD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;qBAAM,CAAC;oBACN,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;oBACtE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,eAAe,IAAI,UAAU,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,6EAA6E;AAC7E,EAAE;AACF,yEAAyE;AACzE,gEAAgE;AAChE,8EAA8E;AAC9E,0EAA0E;AAC1E,sEAAsE;AACtE,2EAA2E;AAC3E,sBAAsB;AAEtB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;AActF,MAAM,kBAAkB;IACb,IAAI,GAAG,WAAoB,CAAC;IAErC,KAAK,CAAC,IAAqB;QACzB,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,KAAK,GAAG,CAAC,EAAE,GAAY;QACzC,OAAO,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;CACF;AAMD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAsC,CAAC;AAEpE,KAAK,UAAU,YAAY,CAAC,IAAqB;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,MAAM,GAAG,GAAG,IAAI,KAAK,YAAY;QAC/B,CAAC,CAAC,MAAM,MAAM,CAAC,mCAAmC,CAAC;QACnD,CAAC,CAAC,MAAM,MAAM,CAAC,oCAAoC,CAAC,CAAC;IACvD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC7B,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY;IACK;IAAwC;IAA7D,YAAqB,IAAqB,EAAmB,QAA2B;QAAnE,SAAI,GAAJ,IAAI,CAAiB;QAAmB,aAAQ,GAAR,QAAQ,CAAmB;IAAG,CAAC;IAE5F,KAAK,CAAC,IAAqB;QACzB,4EAA4E;QAC5E,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,KAAK,GAAG,CAAC,EAAE,GAAY;QACzC,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC7E,MAAM,EAAE,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjG,IAAI,IAAI,IAAI,EAAE;YAAE,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAmB;IACvD,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,kBAAkB,EAAE,CAAC;IAC1D,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,IAAI,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,QAAiB,EACjB,MAA0B,OAAO,CAAC,GAAG,CAAC,aAAa;IAEnD,MAAM,SAAS,GAAG,QAAQ,IAAI,GAAG,CAAC;IAClC,IAAI,SAAS,IAAI,IAAI,IAAI,SAAS,KAAK,EAAE;QAAE,OAAO,WAAW,CAAC;IAC9D,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,uBAAuB,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cesarandreslopez/occ",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.10",
|
|
4
4
|
"description": "Office Cloc and Count — document metrics, structure extraction, content inspection, and code exploration for real repositories",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/src/index.js",
|
|
@@ -348,6 +348,7 @@
|
|
|
348
348
|
"cli-table3": "^0.6.5",
|
|
349
349
|
"commander": "^12.1.0",
|
|
350
350
|
"fast-glob": "^3.3.2",
|
|
351
|
+
"gpt-tokenizer": "^3.4.0",
|
|
351
352
|
"ignore": "^7.0.5",
|
|
352
353
|
"jszip": "^3.10.1",
|
|
353
354
|
"mammoth": "^1.8.0",
|