@paparats/shared 2.1.1 → 2.2.0

package/dist/index.d.ts

@@ -2,6 +2,7 @@ export { validateIndexingPaths } from './path-validation.js';
 export { normalizeExcludePatterns } from './exclude-patterns.js';
 export { LANGUAGE_EXCLUDE_DEFAULTS, COMMON_EXCLUDE, DEFAULT_EXCLUDE_BARE, getDefaultExcludeForLanguages, } from './language-excludes.js';
 export { createGitignoreFilter, filterFilesByGitignore } from './gitignore.js';
+export { detectNonSource, type NonSourceVerdict } from './non-source-detect.js';
 export { detectLanguageByPath, detectProjectLanguage } from './language-detect.js';
 export { DEFAULT_GROUP } from './group.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,KAAK,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -2,6 +2,7 @@ export { validateIndexingPaths } from './path-validation.js';
 export { normalizeExcludePatterns } from './exclude-patterns.js';
 export { LANGUAGE_EXCLUDE_DEFAULTS, COMMON_EXCLUDE, DEFAULT_EXCLUDE_BARE, getDefaultExcludeForLanguages, } from './language-excludes.js';
 export { createGitignoreFilter, filterFilesByGitignore } from './gitignore.js';
+export { detectNonSource } from './non-source-detect.js';
 export { detectLanguageByPath, detectProjectLanguage } from './language-detect.js';
 export { DEFAULT_GROUP } from './group.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EACL,yBAAyB,EACzB,cAAc,EACd,oBAAoB,EACpB,6BAA6B,GAC9B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAyB,MAAM,wBAAwB,CAAC;AAChF,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/language-excludes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"language-excludes.d.ts","sourceRoot":"","sources":["../src/language-excludes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAgH9D,CAAC;AAEF,eAAO,MAAM,cAAc,UAAW,CAAC;AAEvC;;;GAGG;AACH,eAAO,MAAM,oBAAoB,UAiBhC,CAAC;AAEF,wBAAgB,6BAA6B,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAW3E"}
+{"version":3,"file":"language-excludes.d.ts","sourceRoot":"","sources":["../src/language-excludes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CA6H9D,CAAC;AAEF,eAAO,MAAM,cAAc,UAAW,CAAC;AAEvC;;;GAGG;AACH,eAAO,MAAM,oBAAoB,UAiBhC,CAAC;AAEF,wBAAgB,6BAA6B,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAW3E"}

package/dist/language-excludes.js

@@ -33,6 +33,16 @@ export const LANGUAGE_EXCLUDE_DEFAULTS = {
         '.vercel',
         '.swc',
         '*.tsbuildinfo',
+        // Machine-generated / bundled artifacts: source-file extensions but no
+        // source structure (minified bundles, base64 asset blobs). A dense wall of
+        // tokens is worthless for retrieval and pathological for the embedder. The
+        // structural detector (non-source-detect.ts) is the content-based backstop;
+        // these globs skip the obvious classes without reading the file.
+        '**/*.min.js',
+        '**/*.min.css',
+        '**/*.bundle.js',
+        '**/*.data.ts',
+        '**/*.data.js',
     ],
     javascript: [
         'node_modules',
@@ -44,6 +54,9 @@ export const LANGUAGE_EXCLUDE_DEFAULTS = {
         'out',
         '.nuxt',
         '.output',
+        '**/*.min.js',
+        '**/*.min.css',
+        '**/*.bundle.js',
     ],
     python: [
         'venv',

package/dist/language-excludes.js.map

@@ -1 +1 @@
-{"version":3,"file":"language-excludes.js","sourceRoot":"","sources":["../src/language-excludes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAA6B;IACjE,IAAI,EAAE;QACJ,QAAQ;QACR,KAAK;QACL,KAAK;QACL,MAAM;QACN,MAAM;QACN,cAAc;QACd,SAAS;QACT,UAAU;QACV,cAAc;QACd,eAAe;QACf,SAAS;QACT,iBAAiB;QACjB,KAAK;QACL,QAAQ;QACR,UAAU;KACX;IACD,UAAU,EAAE;QACV,cAAc;QACd,MAAM;QACN,OAAO;QACP,UAAU;QACV,OAAO;QACP,QAAQ;QACR,KAAK;QACL,QAAQ;QACR,SAAS;QACT,MAAM;QACN,eAAe;KAChB;IACD,UAAU,EAAE;QACV,cAAc;QACd,MAAM;QACN,UAAU;QACV,OAAO;QACP,QAAQ;QACR,eAAe;QACf,KAAK;QACL,OAAO;QACP,SAAS;KACV;IACD,MAAM,EAAE;QACN,MAAM;QACN,aAAa;QACb,OAAO;QACP,aAAa;QACb,YAAY;QACZ,eAAe;QACf,MAAM;QACN,WAAW;QACX,SAAS;QACT,MAAM;QACN,OAAO;QACP,aAAa;QACb,SAAS;KACV;IACD,EAAE,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC;IACjD,IAAI,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC;IACtD,IAAI,EAAE;QACJ,OAAO;QACP,SAAS;QACT,QAAQ;QACR,KAAK;QACL,OAAO;QACP,YAAY;QACZ,UAAU;QACV,WAAW;QACX,SAAS;QACT,OAAO;QACP,OAAO;KACR;IACD,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,uBAAuB,CAAC;IAC9F,GAAG,EAAE;QACH,OAAO;QACP,eAAe;QACf,KAAK;QACL,MAAM;QACN,KAAK;QACL,OAAO;QACP,OAAO;QACP,uBAAuB;QACvB,aAAa;QACb,SAAS;KACV;IACD,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;IAClF,GAAG,EAAE;QACH,QAAQ;QACR,cAAc;QACd,gBAAgB;QAChB,yBAAyB;QACzB,cAAc;QACd,iBAAiB;QACjB,MAAM;KACP;IACD,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC;IAC5E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,iBAAiB,CAAC;IACnF,MAAM,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC;IAC5D,KAAK,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC;IACpF,+EAA+E;IAC/E,iFAAiF;IACjF,gEAAgE;IAChE,SAAS,EAAE;QACT,YAAY;QACZ,cAAc;QACd,gBAAgB;QAChB,aAAa;QACb,kBAAkB;QAClB,kBAAkB;QAClB,uBAAuB;KACxB;IACD,OAAO,EAAE,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;CACvE,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,cAAc;IACd,iBAAiB;IACjB,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,aAAa;IACb,iBAAiB;CAClB,CAAC;AAEF,MAAM,UAAU,6BAA6B,CAAC,SAAmB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,cAAc,CAAC,CAAC;IAC/C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,QAAQ,GACZ,yBAAyB,CAAC,IAAI,CAAC;YAC/B,yBAAyB,CAAC,UAAU;YACpC,yBAAyB,CAAC,OAAO;YACjC,EAAE,CAAC;QACL,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}
+{"version":3,"file":"language-excludes.js","sourceRoot":"","sources":["../src/language-excludes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAA6B;IACjE,IAAI,EAAE;QACJ,QAAQ;QACR,KAAK;QACL,KAAK;QACL,MAAM;QACN,MAAM;QACN,cAAc;QACd,SAAS;QACT,UAAU;QACV,cAAc;QACd,eAAe;QACf,SAAS;QACT,iBAAiB;QACjB,KAAK;QACL,QAAQ;QACR,UAAU;KACX;IACD,UAAU,EAAE;QACV,cAAc;QACd,MAAM;QACN,OAAO;QACP,UAAU;QACV,OAAO;QACP,QAAQ;QACR,KAAK;QACL,QAAQ;QACR,SAAS;QACT,MAAM;QACN,eAAe;QACf,uEAAuE;QACvE,2EAA2E;QAC3E,2EAA2E;QAC3E,4EAA4E;QAC5E,iEAAiE;QACjE,aAAa;QACb,cAAc;QACd,gBAAgB;QAChB,cAAc;QACd,cAAc;KACf;IACD,UAAU,EAAE;QACV,cAAc;QACd,MAAM;QACN,UAAU;QACV,OAAO;QACP,QAAQ;QACR,eAAe;QACf,KAAK;QACL,OAAO;QACP,SAAS;QACT,aAAa;QACb,cAAc;QACd,gBAAgB;KACjB;IACD,MAAM,EAAE;QACN,MAAM;QACN,aAAa;QACb,OAAO;QACP,aAAa;QACb,YAAY;QACZ,eAAe;QACf,MAAM;QACN,WAAW;QACX,SAAS;QACT,MAAM;QACN,OAAO;QACP,aAAa;QACb,SAAS;KACV;IACD,EAAE,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC;IACjD,IAAI,EAAE,CAAC,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC;IACtD,IAAI,EAAE;QACJ,OAAO;QACP,SAAS;QACT,QAAQ;QACR,KAAK;QACL,OAAO;QACP,YAAY;QACZ,UAAU;QACV,WAAW;QACX,SAAS;QACT,OAAO;QACP,OAAO;KACR;IACD,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,uBAAuB,CAAC;IAC9F,GAAG,EAAE;QACH,OAAO;QACP,eAAe;QACf,KAAK;QACL,MAAM;QACN,KAAK;QACL,OAAO;QACP,OAAO;QACP,uBAAuB;QACvB,aAAa;QACb,SAAS;KACV;IACD,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC;IAClF,GAAG,EAAE;QACH,QAAQ;QACR,cAAc;QACd,gBAAgB;QAChB,yBAAyB;QACzB,cAAc;QACd,iBAAiB;QACjB,MAAM;KACP;IACD,MAAM,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC;IAC5E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,iBAAiB,CAAC;IACnF,MAAM,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC;IAC5D,KAAK,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,CAAC;IACpF,+EAA+E;IAC/E,iFAAiF;IACjF,gEAAgE;IAChE,SAAS,EAAE;QACT,YAAY;QACZ,cAAc;QACd,gBAAgB;QAChB,aAAa;QACb,kBAAkB;QAClB,kBAAkB;QAClB,uBAAuB;KACxB;IACD,OAAO,EAAE,CAAC,cAAc,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;CACvE,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,cAAc;IACd,iBAAiB;IACjB,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,aAAa;IACb,iBAAiB;CAClB,CAAC;AAEF,MAAM,UAAU,6BAA6B,CAAC,SAAmB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,cAAc,CAAC,CAAC;IAC/C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,QAAQ,GACZ,yBAAyB,CAAC,IAAI,CAAC;YAC/B,yBAAyB,CAAC,UAAU;YACpC,yBAAyB,CAAC,OAAO;YACjC,EAAE,CAAC;QACL,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}

package/dist/non-source-detect.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Heuristic detection of non-source content — base64/data-URI blobs, minified
+ * bundles, and other machine-generated text that has a source-file extension but
+ * no source-file structure.
+ *
+ * Why this exists: files like `convex/export/pptx/assets/bg.data.ts` are `.ts`
+ * by name (so language detection routes them to the AST/embedding path) but hold
+ * a single multi-hundred-KB base64 string literal. Embedding that is worthless
+ * for retrieval and pathological for the model — a dense wall of high-entropy
+ * tokens drives llama-server into timeouts/OOM. We detect and skip such content
+ * BEFORE it ever reaches the embedder.
+ *
+ * The signal is structural, not name-based (name globs are a separate, coarser
+ * defence in language-excludes.ts): real source code has short lines, plenty of
+ * whitespace, and a modest alphabet; machine blobs have enormous lines, little
+ * whitespace, and a base64-dominated character distribution. Pure function so it
+ * is trivially testable and identical for the file-level and chunk-level checks.
+ */
+/** Why a piece of content was judged non-source — surfaced in logs/telemetry. */
+export interface NonSourceVerdict {
+    /** True when the content looks machine-generated / non-source. */
+    isNonSource: boolean;
+    /** Human-readable reason (e.g. "max line 780191 chars"). Empty when source. */
+    reason: string;
+}
+/**
+ * Judge whether `content` is non-source machine data unsuitable for embedding.
+ *
+ * Applies to both a whole file (pre-chunking) and an individual chunk
+ * (pre-embedding). Short inputs are always treated as source — the statistics
+ * are unreliable below {@link MIN_LENGTH_TO_ANALYZE} and a false positive there
+ * would drop legitimate small files.
+ */
+export declare function detectNonSource(content: string): NonSourceVerdict;
+//# sourceMappingURL=non-source-detect.d.ts.map

package/dist/non-source-detect.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-source-detect.d.ts","sourceRoot":"","sources":["../src/non-source-detect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,iFAAiF;AACjF,MAAM,WAAW,gBAAgB;IAC/B,kEAAkE;IAClE,WAAW,EAAE,OAAO,CAAC;IACrB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;CAChB;AAqBD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAiDjE"}

package/dist/non-source-detect.js

@@ -0,0 +1,100 @@
+/**
+ * Heuristic detection of non-source content — base64/data-URI blobs, minified
+ * bundles, and other machine-generated text that has a source-file extension but
+ * no source-file structure.
+ *
+ * Why this exists: files like `convex/export/pptx/assets/bg.data.ts` are `.ts`
+ * by name (so language detection routes them to the AST/embedding path) but hold
+ * a single multi-hundred-KB base64 string literal. Embedding that is worthless
+ * for retrieval and pathological for the model — a dense wall of high-entropy
+ * tokens drives llama-server into timeouts/OOM. We detect and skip such content
+ * BEFORE it ever reaches the embedder.
+ *
+ * The signal is structural, not name-based (name globs are a separate, coarser
+ * defence in language-excludes.ts): real source code has short lines, plenty of
+ * whitespace, and a modest alphabet; machine blobs have enormous lines, little
+ * whitespace, and a base64-dominated character distribution. Pure function so it
+ * is trivially testable and identical for the file-level and chunk-level checks.
+ */
+// Thresholds. Deliberately conservative: real source (even generated-but-readable
+// code, long JSON fixtures, SVG paths embedded in JSX) must pass. These fire only
+// on genuinely structureless blobs. Tuning notes live next to each constant.
+/** A single line this long is not hand-written source — minified/one-line blob. */
+const MAX_LINE_LENGTH = 5_000;
+/** Average line length above this means almost no line breaks across the file. */
+const MAX_AVG_LINE_LENGTH = 1_000;
+/** Below this whitespace ratio, text is a dense token wall, not code. */
+const MIN_WHITESPACE_RATIO = 0.02;
+/** Above this base64-alphabet ratio (over a long span), it's an encoded blob. */
+const MAX_BASE64_RATIO = 0.95;
+/** Skip the analysis for short content — the ratios are noisy on tiny inputs. */
+const MIN_LENGTH_TO_ANALYZE = 512;
+// [A-Za-z0-9+/=_-] — the base64 / base64url alphabet plus padding.
+const BASE64_CHAR = /[A-Za-z0-9+/=_-]/;
+const WHITESPACE_CHAR = /\s/;
+/**
+ * Judge whether `content` is non-source machine data unsuitable for embedding.
+ *
+ * Applies to both a whole file (pre-chunking) and an individual chunk
+ * (pre-embedding). Short inputs are always treated as source — the statistics
+ * are unreliable below {@link MIN_LENGTH_TO_ANALYZE} and a false positive there
+ * would drop legitimate small files.
+ */
+export function detectNonSource(content) {
+    const source = () => ({ isNonSource: false, reason: '' });
+    if (content.length < MIN_LENGTH_TO_ANALYZE)
+        return source();
+    // Longest single line — the strongest single signal for minified/one-line blobs.
+    let maxLine = 0;
+    let cur = 0;
+    for (let i = 0; i < content.length; i++) {
+        if (content.charCodeAt(i) === 10 /* \n */) {
+            if (cur > maxLine)
+                maxLine = cur;
+            cur = 0;
+        }
+        else {
+            cur++;
+        }
+    }
+    if (cur > maxLine)
+        maxLine = cur;
+    if (maxLine > MAX_LINE_LENGTH) {
+        return { isNonSource: true, reason: `line of ${maxLine} chars (minified/blob)` };
+    }
+    // Character-distribution pass: whitespace ratio + base64-alphabet dominance.
+    let whitespace = 0;
+    let base64 = 0;
+    for (let i = 0; i < content.length; i++) {
+        const ch = content[i];
+        if (WHITESPACE_CHAR.test(ch))
+            whitespace++;
+        else if (BASE64_CHAR.test(ch))
+            base64++;
+    }
+    const len = content.length;
+    const whitespaceRatio = whitespace / len;
+    const base64Ratio = base64 / len;
+    // Average line length: total length divided by line count.
+    const lineCount = countLines(content);
+    const avgLine = len / lineCount;
+    if (whitespaceRatio < MIN_WHITESPACE_RATIO && base64Ratio > MAX_BASE64_RATIO) {
+        return {
+            isNonSource: true,
+            reason: `dense base64 blob (${(base64Ratio * 100).toFixed(0)}% base64, ${(whitespaceRatio * 100).toFixed(1)}% whitespace)`,
+        };
+    }
+    if (avgLine > MAX_AVG_LINE_LENGTH) {
+        return { isNonSource: true, reason: `avg line ${avgLine.toFixed(0)} chars (no structure)` };
+    }
+    return source();
+}
+function countLines(content) {
+    let n = 1;
+    for (let i = 0; i < content.length; i++) {
+        if (content.charCodeAt(i) === 10)
+            n++;
+    }
+    return n;
+}
+//# sourceMappingURL=non-source-detect.js.map

package/dist/non-source-detect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-source-detect.js","sourceRoot":"","sources":["../src/non-source-detect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAUH,kFAAkF;AAClF,kFAAkF;AAClF,6EAA6E;AAE7E,mFAAmF;AACnF,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,kFAAkF;AAClF,MAAM,mBAAmB,GAAG,KAAK,CAAC;AAClC,yEAAyE;AACzE,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,iFAAiF;AACjF,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,iFAAiF;AACjF,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,mEAAmE;AACnE,MAAM,WAAW,GAAG,kBAAkB,CAAC;AACvC,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,MAAM,GAAG,GAAqB,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,MAAM,GAAG,qBAAqB;QAAE,OAAO,MAAM,EAAE,CAAC;IAE5D,iFAAiF;IACjF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,CAAC;YAC1C,IAAI,GAAG,GAAG,OAAO;gBAAE,OAAO,GAAG,GAAG,CAAC;YACjC,GAAG,GAAG,CAAC,CAAC;QACV,CAAC;aAAM,CAAC;YACN,GAAG,EAAE,CAAC;QACR,CAAC;IACH,CAAC;IACD,IAAI,GAAG,GAAG,OAAO;QAAE,OAAO,GAAG,GAAG,CAAC;IACjC,IAAI,OAAO,GAAG,eAAe,EAAE,CAAC;QAC9B,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,OAAO,wBAAwB,EAAE,CAAC;IACnF,CAAC;IAED,6EAA6E;IAC7E,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;QACvB,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,UAAU,EAAE,CAAC;aACtC,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,EAAE,CAAC;IAC1C,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3B,MAAM,eAAe,GAAG,UAAU,GAAG,GAAG,CAAC;IACzC,MAAM,WAAW,GAAG,MAAM,GAAG,GAAG,CAAC;IAEjC,2DAA2D;IAC3D,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,GAAG,GAAG,SAAS,CAAC;IAEhC,IAAI,eAAe,GAAG,oBAAoB,IAAI,WAAW,GAAG,gBAAgB,EAAE,CAAC;QAC7E,OAAO;YACL,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,sBAAsB,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CACvE,eAAe,GAAG,GAAG,CACtB,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;SAC5B,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,GAAG,mBAAmB,EAAE,CAAC;QAClC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,EAAE,CAAC;IAC9F,CAAC;IAED,OAAO,MAAM,EAAE,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,OAAe;IACjC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE;YAAE,CAAC,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}

package/dist/non-source-detect.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=non-source-detect.test.d.ts.map

package/dist/non-source-detect.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-source-detect.test.d.ts","sourceRoot":"","sources":["../src/non-source-detect.test.ts"],"names":[],"mappings":""}

package/dist/non-source-detect.test.js

@@ -0,0 +1,76 @@
+import { describe, it, expect } from 'vitest';
+import { detectNonSource } from './non-source-detect.js';
+// A base64-ish string of the requested length — the shape of the pptx asset
+// blobs that stalled the embedder (convex/export/pptx/assets/*.data.ts).
+function base64Blob(len) {
+    const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+    let out = '';
+    for (let i = 0; i < len; i++)
+        out += alphabet[i % alphabet.length];
+    return out;
+}
+describe('detectNonSource — flags machine/non-source content', () => {
+    it('flags a single-line base64 asset blob (the .data.ts case)', () => {
+        const content = `export const asset = "${base64Blob(400_000)}";`;
+        const v = detectNonSource(content);
+        expect(v.isNonSource).toBe(true);
+        expect(v.reason).toMatch(/minified|blob|base64|structure/);
+    });
+    it('flags a data-URI export', () => {
+        const content = `export default "data:image/png;base64,${base64Blob(50_000)}";`;
+        expect(detectNonSource(content).isNonSource).toBe(true);
+    });
+    it('flags minified JS (one enormous line)', () => {
+        const min = `!function(){var a=${base64Blob(20_000)};return a}();`;
+        expect(detectNonSource(min).isNonSource).toBe(true);
+    });
+    it('flags a dense base64 wall even with occasional newlines', () => {
+        // 200 lines of 300 base64 chars each: high base64 ratio, low whitespace,
+        // but no single line trips MAX_LINE_LENGTH — the distribution pass catches it.
+        const lines = Array.from({ length: 200 }, () => base64Blob(300));
+        expect(detectNonSource(lines.join('\n')).isNonSource).toBe(true);
+    });
+});
+describe('detectNonSource — passes legitimate source', () => {
+    it('passes normal TypeScript', () => {
+        const ts = `
+import { foo } from './foo.js';
+
+export function greet(name: string): string {
+  // Return a friendly greeting.
+  return \`Hello, \${name}!\`;
+}
+
+export class Service {
+  constructor(private readonly deps: Deps) {}
+  async run(): Promise<void> {
+    for (const item of this.deps.items) {
+      await this.process(item);
+    }
+  }
+}
+`.repeat(10);
+        expect(detectNonSource(ts).isNonSource).toBe(false);
+    });
+    it('passes a large but well-structured JSON fixture', () => {
+        const rows = Array.from({ length: 500 }, (_, i) => `  { "id": ${i}, "name": "item ${i}" }`);
+        const json = `[\n${rows.join(',\n')}\n]`;
+        expect(detectNonSource(json).isNonSource).toBe(false);
+    });
+    it('passes code with an embedded SVG path (long-ish but whitespaced)', () => {
+        const jsx = `
+export const Icon = () => (
+  <svg viewBox="0 0 24 24">
+    <path d="M12 2L2 7v10l10 5 10-5V7z M12 4.3l6 3v6.4l-6 3-6-3V7.3z" />
+  </svg>
+);
+`.repeat(20);
+        expect(detectNonSource(jsx).isNonSource).toBe(false);
+    });
+    it('always treats short content as source (statistics unreliable below threshold)', () => {
+        expect(detectNonSource('a').isNonSource).toBe(false);
+        expect(detectNonSource(base64Blob(100)).isNonSource).toBe(false);
+        expect(detectNonSource('').isNonSource).toBe(false);
+    });
+});
+//# sourceMappingURL=non-source-detect.test.js.map

package/dist/non-source-detect.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"non-source-detect.test.js","sourceRoot":"","sources":["../src/non-source-detect.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,4EAA4E;AAC5E,yEAAyE;AACzE,SAAS,UAAU,CAAC,GAAW;IAC7B,MAAM,QAAQ,GAAG,kEAAkE,CAAC;IACpF,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,GAAG,IAAI,QAAQ,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,QAAQ,CAAC,oDAAoD,EAAE,GAAG,EAAE;IAClE,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;QACnE,MAAM,OAAO,GAAG,yBAAyB,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC;QACjE,MAAM,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACnC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,OAAO,GAAG,yCAAyC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;QAChF,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,GAAG,GAAG,qBAAqB,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC;QACnE,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,yEAAyE;QACzE,+EAA+E;QAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,4CAA4C,EAAE,GAAG,EAAE;IAC1D,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;QAClC,MAAM,EAAE,GAAG;;;;;;;;;;;;;;;;CAgBd,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACT,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;QACzD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC5F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;QACzC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QAC1E,MAAM,GAAG,GAAG;;;;;;CAMf,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACT,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+EAA+E,EAAE,GAAG,EAAE;QACvF,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@paparats/shared",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "type": "module",
   "description": "Shared utilities for Paparats MCP - path validation, gitignore filtering, language-aware exclude patterns",
   "license": "MIT",