@shrkcrft/context 0.1.0-alpha.1 → 0.1.0-alpha.11

package/README.md

@@ -2,7 +2,7 @@
 
 SharkCraft AI context builder: token-budgeted relevance retrieval for tasks.
 
-Part of [SharkCraft](https://github.com/sharkcraft/sharkcraft) — a deterministic, local-first toolkit that gives AI coding agents durable project context. See the main repo for documentation, examples, and the `shrk` CLI.
+Part of [SharkCraft](https://github.com/shrkcrft/sharkcraft) — a deterministic, local-first toolkit that gives AI coding agents durable project context. See the main repo for documentation, examples, and the `shrk` CLI.
 
 ## Install
 

package/dist/context-builder.js

@@ -2,7 +2,7 @@ import { aggregateActionHints, formatAggregatedHints } from '@shrkcrft/knowledge
 import { DEFAULT_CONTEXT_REQUEST } from "./context-request.js";
 import { selectRelevantEntries } from "./relevance-selector.js";
 import { formatEntryForContext, formatSectionBody } from "./ai-context-formatter.js";
-import { countTokens } from "./tokenizer.js";
+import { estimateTokens } from "./token-estimator.js";
 export function buildContext(allEntries, request) {
     const r = { ...DEFAULT_CONTEXT_REQUEST, ...request };
     const maxTokens = r.maxTokens ?? DEFAULT_CONTEXT_REQUEST.maxTokens;
@@ -43,7 +43,7 @@ export function buildContext(allEntries, request) {
     const omitted = [];
     let used = 0;
     function tryAddSection(title, body, entryIds) {
-        const tokens = countTokens(body);
+        const tokens = estimateTokens(body);
         if (used + tokens > maxTokens && sections.length > 0) {
             omitted.push(title);
             return;
@@ -52,7 +52,7 @@ export function buildContext(allEntries, request) {
             // Still emit, but mark truncated.
             const ratio = (maxTokens - used) / tokens;
             const truncatedBody = body.slice(0, Math.max(0, Math.floor(body.length * ratio))) + '\n…[truncated]';
-            const truncTokens = countTokens(truncatedBody);
+            const truncTokens = estimateTokens(truncatedBody);
             sections.push({ title, body: truncatedBody, entryIds, tokens: truncTokens, truncated: true });
             used += truncTokens;
             return;

package/dist/index.d.ts

@@ -3,7 +3,6 @@ export * from './context-result.js';
 export * from './context-section.js';
 export * from './context-builder.js';
 export * from './token-estimator.js';
-export * from './tokenizer.js';
 export * from './relevance-selector.js';
 export * from './ai-context-formatter.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC"}

package/dist/index.js

@@ -3,6 +3,5 @@ export * from "./context-result.js";
 export * from "./context-section.js";
 export * from "./context-builder.js";
 export * from "./token-estimator.js";
-export * from "./tokenizer.js";
 export * from "./relevance-selector.js";
 export * from "./ai-context-formatter.js";

package/package.json

@@ -1,15 +1,16 @@
 {
   "name": "@shrkcrft/context",
-  "version": "0.1.0-alpha.1",
+  "version": "0.1.0-alpha.11",
   "description": "SharkCraft AI context builder: token-budgeted relevance retrieval for tasks.",
   "license": "MIT",
   "author": "SharkCraft contributors",
   "type": "module",
   "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "types": "./dist/index.d.d.ts",
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
+      "bun": "./src/index.ts",
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     }
@@ -21,12 +22,12 @@
   ],
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/sharkcraft/sharkcraft.git",
+    "url": "git+https://github.com/shrkcrft/sharkcraft.git",
     "directory": "packages/context"
   },
-  "homepage": "https://github.com/sharkcraft/sharkcraft",
+  "homepage": "https://github.com/shrkcrft/sharkcraft",
   "bugs": {
-    "url": "https://github.com/sharkcraft/sharkcraft/issues"
+    "url": "https://github.com/shrkcrft/sharkcraft/issues"
   },
   "keywords": [
     "sharkcraft",
@@ -43,12 +44,11 @@
     "typecheck": "tsc --noEmit -p tsconfig.json"
   },
   "dependencies": {
-    "@shrkcrft/core": "^0.1.0-alpha.1",
-    "@shrkcrft/knowledge": "^0.1.0-alpha.1",
-    "@shrkcrft/paths": "^0.1.0-alpha.1",
-    "@shrkcrft/rules": "^0.1.0-alpha.1",
-    "@shrkcrft/templates": "^0.1.0-alpha.1",
-    "gpt-tokenizer": "^3.4.0"
+    "@shrkcrft/core": "^0.1.0-alpha.11",
+    "@shrkcrft/knowledge": "^0.1.0-alpha.11",
+    "@shrkcrft/rules": "^0.1.0-alpha.11",
+    "@shrkcrft/paths": "^0.1.0-alpha.11",
+    "@shrkcrft/templates": "^0.1.0-alpha.11"
   },
   "publishConfig": {
     "access": "public"

package/dist/tokenizer.d.ts

@@ -1,29 +0,0 @@
-export interface ITokenizer {
-    /** Human-readable name, surfaced in diagnostics. */
-    readonly name: string;
-    /** Total token count for `text`. Must be deterministic for a given input. */
-    countTokens(text: string): number;
-}
-export declare function getTokenizer(): ITokenizer;
-export declare function setTokenizer(tokenizer: ITokenizer): void;
-export declare function resetTokenizer(): void;
-export declare function countTokens(text: string): number;
-export interface ITokenizerInitOptions {
-    /**
-     * Which BPE encoding to load. Defaults to `cl100k_base` (GPT-4 / GPT-3.5);
-     * pass `o200k_base` for GPT-4o-class models.
-     */
-    encoding?: 'cl100k_base' | 'o200k_base';
-    /**
-     * If true, throw on load failure instead of falling back silently. Useful
-     * in tests; not recommended in production paths.
-     */
-    strict?: boolean;
-}
-/**
- * Asynchronously upgrade the active tokenizer from the estimator to a real
- * BPE encoder. Resolves to `true` if the swap happened; resolves to `false`
- * (or rejects, when `strict`) if the tokenizer module could not be loaded.
- */
-export declare function initTokenizer(options?: ITokenizerInitOptions): Promise<boolean>;
-//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6EAA6E;IAC7E,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC;AAyBD,wBAAgB,YAAY,IAAI,UAAU,CAEzC;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,UAAU,GAAG,IAAI,CAExD;AAED,wBAAgB,cAAc,IAAI,IAAI,CAErC;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,MAAM,WAAW,qBAAqB;IACpC;;;OAGG;IACH,QAAQ,CAAC,EAAE,aAAa,GAAG,YAAY,CAAC;IACxC;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,OAAO,CAAC,CAgBzF"}

package/dist/tokenizer.js

@@ -1,80 +0,0 @@
-var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
-    if (typeof path === "string" && /^\.\.?\//.test(path)) {
-        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
-            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
-        });
-    }
-    return path;
-};
-/**
- * Token counting abstraction.
- *
- * The context builder needs to fit content under a budget — wrong counts mean
- * either truncated context (under-counted) or wasted budget (over-counted).
- * Real BPE tokenizers are accurate; the legacy `estimateTokens` is a 4-char
- * heuristic kept as a fallback so the package still loads in environments
- * where the tokenizer can't be imported.
- *
- * Default flow:
- *   1. On import, the estimator is the active tokenizer (always works).
- *   2. Call `await initTokenizer()` once at startup to swap in the real
- *      tokenizer. If that fails for any reason, the estimator stays active.
- *   3. `countTokens(text)` always returns a number, synchronously.
- */
-import { estimateTokens } from "./token-estimator.js";
-class EstimatorTokenizer {
-    name = 'estimator';
-    countTokens(text) {
-        return estimateTokens(text);
-    }
-}
-class GptTokenizer {
-    encode;
-    name;
-    constructor(name, encode) {
-        this.encode = encode;
-        this.name = name;
-    }
-    countTokens(text) {
-        if (!text)
-            return 0;
-        return this.encode(text).length;
-    }
-}
-let activeTokenizer = new EstimatorTokenizer();
-export function getTokenizer() {
-    return activeTokenizer;
-}
-export function setTokenizer(tokenizer) {
-    activeTokenizer = tokenizer;
-}
-export function resetTokenizer() {
-    activeTokenizer = new EstimatorTokenizer();
-}
-export function countTokens(text) {
-    return activeTokenizer.countTokens(text);
-}
-/**
- * Asynchronously upgrade the active tokenizer from the estimator to a real
- * BPE encoder. Resolves to `true` if the swap happened; resolves to `false`
- * (or rejects, when `strict`) if the tokenizer module could not be loaded.
- */
-export async function initTokenizer(options = {}) {
-    const encoding = options.encoding ?? 'cl100k_base';
-    try {
-        const moduleName = encoding === 'o200k_base' ? 'gpt-tokenizer/encoding/o200k_base' : 'gpt-tokenizer/encoding/cl100k_base';
-        const mod = (await import(__rewriteRelativeImportExtension(moduleName)));
-        if (typeof mod.encode !== 'function') {
-            if (options.strict)
-                throw new Error(`gpt-tokenizer module ${moduleName} missing encode()`);
-            return false;
-        }
-        setTokenizer(new GptTokenizer(`gpt-tokenizer:${encoding}`, mod.encode));
-        return true;
-    }
-    catch (err) {
-        if (options.strict)
-            throw err;
-        return false;
-    }
-}