@utilix-tech/sdk 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai_agent-CcpkV2DR.d.cts +393 -0
- package/dist/ai_agent-CcpkV2DR.d.ts +393 -0
- package/dist/chunk-M35VJETW.js +953 -0
- package/dist/{chunk-ZKL2VX2G.js → chunk-NSOARQCM.js} +68 -1
- package/dist/{chunk-L6KCTHUW.js → chunk-OKSWDVOM.js} +133 -2
- package/dist/{code-QNrdLIR3.d.cts → code-BUqyaofO.d.cts} +18 -2
- package/dist/{code-QNrdLIR3.d.ts → code-BUqyaofO.d.ts} +18 -2
- package/dist/index.cjs +1158 -10
- package/dist/index.d.cts +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -2
- package/dist/{misc-DcVBManm.d.cts → misc-CA3N198T.d.cts} +67 -2
- package/dist/{misc-DcVBManm.d.ts → misc-CA3N198T.d.ts} +67 -2
- package/dist/tools/ai_agent.cjs +949 -0
- package/dist/tools/ai_agent.d.cts +1 -0
- package/dist/tools/ai_agent.d.ts +1 -0
- package/dist/tools/ai_agent.js +2 -0
- package/dist/tools/code.cjs +67 -0
- package/dist/tools/code.d.cts +1 -1
- package/dist/tools/code.d.ts +1 -1
- package/dist/tools/code.js +1 -1
- package/dist/tools/misc.cjs +136 -0
- package/dist/tools/misc.d.cts +1 -1
- package/dist/tools/misc.d.ts +1 -1
- package/dist/tools/misc.js +1 -1
- package/package.json +6 -1
package/dist/index.cjs
CHANGED
|
@@ -9798,10 +9798,13 @@ __export(code_exports, {
|
|
|
9798
9798
|
buildGitCommand: () => buildGitCommand,
|
|
9799
9799
|
collapseHtmlWhitespace: () => collapseHtmlWhitespace,
|
|
9800
9800
|
collapseJsWhitespace: () => collapseJsWhitespace,
|
|
9801
|
+
countProperties: () => countProperties,
|
|
9802
|
+
countRules: () => countRules,
|
|
9801
9803
|
detectDialect: () => detectDialect,
|
|
9802
9804
|
detectOperationType: () => detectOperationType,
|
|
9803
9805
|
envToExport: () => envToExport2,
|
|
9804
9806
|
envToJson: () => envToJson2,
|
|
9807
|
+
formatCss: () => formatCss,
|
|
9805
9808
|
formatGql: () => formatGql,
|
|
9806
9809
|
formatHtml: () => formatHtml,
|
|
9807
9810
|
formatSql: () => formatSql,
|
|
@@ -9810,6 +9813,7 @@ __export(code_exports, {
|
|
|
9810
9813
|
getJsStats: () => getJsStats,
|
|
9811
9814
|
getMethodColor: () => getMethodColor,
|
|
9812
9815
|
isValidTag: () => isValidTag,
|
|
9816
|
+
minifyCss: () => minifyCss,
|
|
9813
9817
|
minifyGql: () => minifyGql,
|
|
9814
9818
|
minifyHtml: () => minifyHtml,
|
|
9815
9819
|
minifyJs: () => minifyJs,
|
|
@@ -11996,6 +12000,69 @@ function envToExport2(input) {
|
|
|
11996
12000
|
function validateEnvKey2(key) {
|
|
11997
12001
|
return /^[A-Za-z_][A-Za-z0-9_]*$/.test(key);
|
|
11998
12002
|
}
|
|
12003
|
+
function _calcCssResult(input, output) {
|
|
12004
|
+
const originalSize = input.length;
|
|
12005
|
+
const outputSize = output.length;
|
|
12006
|
+
const savingsPercent = originalSize === 0 ? 0 : Math.round((originalSize - outputSize) / originalSize * 100 * 100) / 100;
|
|
12007
|
+
return { output, originalSize, outputSize, savingsPercent };
|
|
12008
|
+
}
|
|
12009
|
+
function minifyCss(input) {
|
|
12010
|
+
let output = input;
|
|
12011
|
+
output = output.replace(/\/\*[\s\S]*?\*\//g, "");
|
|
12012
|
+
output = output.replace(/\s+/g, " ");
|
|
12013
|
+
output = output.replace(/\s*{\s*/g, "{");
|
|
12014
|
+
output = output.replace(/\s*}\s*/g, "}");
|
|
12015
|
+
output = output.replace(/\s*:\s*/g, ":");
|
|
12016
|
+
output = output.replace(/\s*;\s*/g, ";");
|
|
12017
|
+
output = output.replace(/\s*,\s*/g, ",");
|
|
12018
|
+
output = output.replace(/;}/g, "}");
|
|
12019
|
+
output = output.trim();
|
|
12020
|
+
return _calcCssResult(input, output);
|
|
12021
|
+
}
|
|
12022
|
+
function formatCss(input, indentSize = 2) {
|
|
12023
|
+
const indent = " ".repeat(indentSize);
|
|
12024
|
+
const { output: minified } = minifyCss(input);
|
|
12025
|
+
let result = "";
|
|
12026
|
+
let insideBlock = false;
|
|
12027
|
+
let i = 0;
|
|
12028
|
+
while (i < minified.length) {
|
|
12029
|
+
const char = minified[i];
|
|
12030
|
+
if (char === "{") {
|
|
12031
|
+
result += " {\n";
|
|
12032
|
+
insideBlock = true;
|
|
12033
|
+
} else if (char === "}") {
|
|
12034
|
+
result = result.trimEnd() + "\n";
|
|
12035
|
+
result += "}\n\n";
|
|
12036
|
+
insideBlock = false;
|
|
12037
|
+
} else if (char === ";" && insideBlock) {
|
|
12038
|
+
result += ";\n";
|
|
12039
|
+
if (i + 1 < minified.length && minified[i + 1] !== "}") result += indent;
|
|
12040
|
+
} else {
|
|
12041
|
+
if (insideBlock && result.endsWith("{\n")) result += indent;
|
|
12042
|
+
result += char;
|
|
12043
|
+
}
|
|
12044
|
+
i++;
|
|
12045
|
+
}
|
|
12046
|
+
return _calcCssResult(input, result.trimEnd());
|
|
12047
|
+
}
|
|
12048
|
+
function countRules(css) {
|
|
12049
|
+
return (css.match(/{/g) ?? []).length;
|
|
12050
|
+
}
|
|
12051
|
+
function countProperties(css) {
|
|
12052
|
+
let count = 0;
|
|
12053
|
+
let insideBlock = false;
|
|
12054
|
+
for (const char of css) {
|
|
12055
|
+
if (char === "{") insideBlock = true;
|
|
12056
|
+
else if (char === "}") insideBlock = false;
|
|
12057
|
+
else if (char === ";" && insideBlock) count++;
|
|
12058
|
+
}
|
|
12059
|
+
css.replace(/{[^{}]*}/g, (block) => {
|
|
12060
|
+
const last = block.slice(1, -1).trim().split(";").pop()?.trim();
|
|
12061
|
+
if (last?.includes(":")) count++;
|
|
12062
|
+
return block;
|
|
12063
|
+
});
|
|
12064
|
+
return count;
|
|
12065
|
+
}
|
|
11999
12066
|
|
|
12000
12067
|
// src/tools/color.ts
|
|
12001
12068
|
var color_exports = {};
|
|
@@ -13139,10 +13206,10 @@ __export(css_exports, {
|
|
|
13139
13206
|
calcClamp: () => calcClamp,
|
|
13140
13207
|
calcSpecificity: () => calcSpecificity,
|
|
13141
13208
|
compareSpecificity: () => compareSpecificity,
|
|
13142
|
-
countProperties: () =>
|
|
13143
|
-
countRules: () =>
|
|
13209
|
+
countProperties: () => countProperties2,
|
|
13210
|
+
countRules: () => countRules2,
|
|
13144
13211
|
explainSelector: () => explainSelector,
|
|
13145
|
-
formatCss: () =>
|
|
13212
|
+
formatCss: () => formatCss2,
|
|
13146
13213
|
generateAnimationCSS: () => generateAnimationCSS,
|
|
13147
13214
|
generateBorderRadius: () => generateBorderRadius,
|
|
13148
13215
|
generateBoxShadow: () => generateBoxShadow,
|
|
@@ -13156,7 +13223,7 @@ __export(css_exports, {
|
|
|
13156
13223
|
generateKeyframesCSS: () => generateKeyframesCSS,
|
|
13157
13224
|
getCurvePoints: () => getCurvePoints,
|
|
13158
13225
|
isUniform: () => isUniform,
|
|
13159
|
-
minifyCss: () =>
|
|
13226
|
+
minifyCss: () => minifyCss2,
|
|
13160
13227
|
multipleSelectors: () => multipleSelectors,
|
|
13161
13228
|
parseBorderRadius: () => parseBorderRadius,
|
|
13162
13229
|
parseBoxShadow: () => parseBoxShadow,
|
|
@@ -14032,7 +14099,7 @@ function calcCssResult(input, output) {
|
|
|
14032
14099
|
const savingsPercent = originalSize === 0 ? 0 : Math.round((originalSize - outputSize) / originalSize * 100 * 100) / 100;
|
|
14033
14100
|
return { output, originalSize, outputSize, savingsPercent };
|
|
14034
14101
|
}
|
|
14035
|
-
function
|
|
14102
|
+
function minifyCss2(input) {
|
|
14036
14103
|
let output = input;
|
|
14037
14104
|
output = output.replace(/\/\*[\s\S]*?\*\//g, "");
|
|
14038
14105
|
output = output.replace(/\s+/g, " ");
|
|
@@ -14045,9 +14112,9 @@ function minifyCss(input) {
|
|
|
14045
14112
|
output = output.trim();
|
|
14046
14113
|
return calcCssResult(input, output);
|
|
14047
14114
|
}
|
|
14048
|
-
function
|
|
14115
|
+
function formatCss2(input, indentSize = 2) {
|
|
14049
14116
|
const indent = " ".repeat(indentSize);
|
|
14050
|
-
const { output: minified } =
|
|
14117
|
+
const { output: minified } = minifyCss2(input);
|
|
14051
14118
|
let result = "";
|
|
14052
14119
|
let insideBlock = false;
|
|
14053
14120
|
let i = 0;
|
|
@@ -14079,11 +14146,11 @@ function formatCss(input, indentSize = 2) {
|
|
|
14079
14146
|
const output = result.trimEnd();
|
|
14080
14147
|
return calcCssResult(input, output);
|
|
14081
14148
|
}
|
|
14082
|
-
function
|
|
14149
|
+
function countRules2(css) {
|
|
14083
14150
|
const matches = css.match(/{/g);
|
|
14084
14151
|
return matches ? matches.length : 0;
|
|
14085
14152
|
}
|
|
14086
|
-
function
|
|
14153
|
+
function countProperties2(css) {
|
|
14087
14154
|
let count = 0;
|
|
14088
14155
|
let insideBlock = false;
|
|
14089
14156
|
for (let i = 0; i < css.length; i++) {
|
|
@@ -14369,6 +14436,7 @@ __export(misc_exports, {
|
|
|
14369
14436
|
COLOR_PRESETS: () => COLOR_PRESETS,
|
|
14370
14437
|
DATA_TYPES: () => DATA_TYPES2,
|
|
14371
14438
|
DEFAULT_CONFIG: () => DEFAULT_CONFIG2,
|
|
14439
|
+
DEFAULT_QR_OPTIONS: () => DEFAULT_QR_OPTIONS2,
|
|
14372
14440
|
FORMAT_EXTENSIONS: () => FORMAT_EXTENSIONS,
|
|
14373
14441
|
FORMAT_LABELS: () => FORMAT_LABELS,
|
|
14374
14442
|
analyzeString: () => analyzeString,
|
|
@@ -14384,6 +14452,9 @@ __export(misc_exports, {
|
|
|
14384
14452
|
formatSvg: () => formatSvg,
|
|
14385
14453
|
fromUnicodeEscape: () => fromUnicodeEscape,
|
|
14386
14454
|
generateData: () => generateData2,
|
|
14455
|
+
generateMetaHtml: () => generateMetaHtml2,
|
|
14456
|
+
generateQrDataUrl: () => generateQrDataUrl2,
|
|
14457
|
+
generateQrSvg: () => generateQrSvg2,
|
|
14387
14458
|
generateSingle: () => generateSingle2,
|
|
14388
14459
|
generateSvgFavicon: () => generateSvgFavicon,
|
|
14389
14460
|
getConversionWarnings: () => getConversionWarnings,
|
|
@@ -14397,10 +14468,12 @@ __export(misc_exports, {
|
|
|
14397
14468
|
numberToWords: () => numberToWords2,
|
|
14398
14469
|
optimizeSvg: () => optimizeSvg,
|
|
14399
14470
|
ordinalWords: () => ordinalWords2,
|
|
14471
|
+
parseMetaTags: () => parseMetaTags2,
|
|
14400
14472
|
romanToNumber: () => romanToNumber2,
|
|
14401
14473
|
sanitizeSvg: () => sanitizeSvg,
|
|
14402
14474
|
svgToDataUrl: () => svgToDataUrl,
|
|
14403
|
-
toUnicodeEscape: () => toUnicodeEscape
|
|
14475
|
+
toUnicodeEscape: () => toUnicodeEscape,
|
|
14476
|
+
validateOgTags: () => validateOgTags2
|
|
14404
14477
|
});
|
|
14405
14478
|
var NAMED_HTML_ENTITIES = {
|
|
14406
14479
|
34: """,
|
|
@@ -15445,10 +15518,1085 @@ function generateData2(config) {
|
|
|
15445
15518
|
}
|
|
15446
15519
|
return results;
|
|
15447
15520
|
}
|
|
15521
|
+
var DEFAULT_QR_OPTIONS2 = {
|
|
15522
|
+
errorLevel: "M",
|
|
15523
|
+
size: 256,
|
|
15524
|
+
margin: 1,
|
|
15525
|
+
darkColor: "#000000",
|
|
15526
|
+
lightColor: "#ffffff"
|
|
15527
|
+
};
|
|
15528
|
+
async function generateQrSvg2(text, options) {
|
|
15529
|
+
const o = { ...DEFAULT_QR_OPTIONS2, ...options };
|
|
15530
|
+
return QRCode__default.default.toString(text, {
|
|
15531
|
+
type: "svg",
|
|
15532
|
+
errorCorrectionLevel: o.errorLevel,
|
|
15533
|
+
margin: o.margin,
|
|
15534
|
+
color: { dark: o.darkColor, light: o.lightColor }
|
|
15535
|
+
});
|
|
15536
|
+
}
|
|
15537
|
+
async function generateQrDataUrl2(text, options) {
|
|
15538
|
+
const o = { ...DEFAULT_QR_OPTIONS2, ...options };
|
|
15539
|
+
return QRCode__default.default.toDataURL(text, {
|
|
15540
|
+
width: o.size,
|
|
15541
|
+
errorCorrectionLevel: o.errorLevel,
|
|
15542
|
+
margin: o.margin,
|
|
15543
|
+
color: { dark: o.darkColor, light: o.lightColor }
|
|
15544
|
+
});
|
|
15545
|
+
}
|
|
15546
|
+
function _getMetaContent(html, property) {
|
|
15547
|
+
const esc = property.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
15548
|
+
const patterns = [
|
|
15549
|
+
new RegExp(`<meta[^>]+property=["']${esc}["'][^>]+content=["']([^"']*)["']`, "i"),
|
|
15550
|
+
new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+property=["']${esc}["']`, "i"),
|
|
15551
|
+
new RegExp(`<meta[^>]+name=["']${esc}["'][^>]+content=["']([^"']*)["']`, "i"),
|
|
15552
|
+
new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+name=["']${esc}["']`, "i")
|
|
15553
|
+
];
|
|
15554
|
+
for (const p of patterns) {
|
|
15555
|
+
const m = html.match(p);
|
|
15556
|
+
if (m) return m[1];
|
|
15557
|
+
}
|
|
15558
|
+
return void 0;
|
|
15559
|
+
}
|
|
15560
|
+
function parseMetaTags2(html) {
|
|
15561
|
+
const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i);
|
|
15562
|
+
const canonicalMatch = html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']*)["']/i) || html.match(/<link[^>]+href=["']([^"']*)["'][^>]+rel=["']canonical["']/i);
|
|
15563
|
+
const charsetMatch = html.match(/<meta[^>]+charset=["']([^"']*)["']/i) || html.match(/<meta[^>]+charset=([^\s>]*)/i);
|
|
15564
|
+
return {
|
|
15565
|
+
og: {
|
|
15566
|
+
title: _getMetaContent(html, "og:title"),
|
|
15567
|
+
description: _getMetaContent(html, "og:description"),
|
|
15568
|
+
image: _getMetaContent(html, "og:image"),
|
|
15569
|
+
imageAlt: _getMetaContent(html, "og:image:alt"),
|
|
15570
|
+
imageWidth: _getMetaContent(html, "og:image:width"),
|
|
15571
|
+
imageHeight: _getMetaContent(html, "og:image:height"),
|
|
15572
|
+
url: _getMetaContent(html, "og:url"),
|
|
15573
|
+
type: _getMetaContent(html, "og:type"),
|
|
15574
|
+
siteName: _getMetaContent(html, "og:site_name"),
|
|
15575
|
+
locale: _getMetaContent(html, "og:locale")
|
|
15576
|
+
},
|
|
15577
|
+
twitter: {
|
|
15578
|
+
card: _getMetaContent(html, "twitter:card"),
|
|
15579
|
+
site: _getMetaContent(html, "twitter:site"),
|
|
15580
|
+
creator: _getMetaContent(html, "twitter:creator"),
|
|
15581
|
+
title: _getMetaContent(html, "twitter:title"),
|
|
15582
|
+
description: _getMetaContent(html, "twitter:description"),
|
|
15583
|
+
image: _getMetaContent(html, "twitter:image")
|
|
15584
|
+
},
|
|
15585
|
+
standard: {
|
|
15586
|
+
title: titleMatch ? titleMatch[1].trim() : void 0,
|
|
15587
|
+
description: _getMetaContent(html, "description"),
|
|
15588
|
+
keywords: _getMetaContent(html, "keywords"),
|
|
15589
|
+
canonical: canonicalMatch ? canonicalMatch[1] : void 0,
|
|
15590
|
+
robots: _getMetaContent(html, "robots"),
|
|
15591
|
+
viewport: _getMetaContent(html, "viewport"),
|
|
15592
|
+
charset: charsetMatch ? charsetMatch[1] : void 0
|
|
15593
|
+
}
|
|
15594
|
+
};
|
|
15595
|
+
}
|
|
15596
|
+
function validateOgTags2(tags) {
|
|
15597
|
+
const warnings = [];
|
|
15598
|
+
const missing = [];
|
|
15599
|
+
const required = ["title", "description", "image", "url", "type"];
|
|
15600
|
+
for (const field of required) {
|
|
15601
|
+
if (!tags[field]) missing.push(`og:${field}`);
|
|
15602
|
+
}
|
|
15603
|
+
if (tags.image && !/^https?:\/\//i.test(tags.image))
|
|
15604
|
+
warnings.push("og:image should be an absolute URL");
|
|
15605
|
+
if (tags.url && !/^https?:\/\//i.test(tags.url))
|
|
15606
|
+
warnings.push("og:url should be an absolute URL");
|
|
15607
|
+
if (tags.title && tags.title.length > 95)
|
|
15608
|
+
warnings.push("og:title is longer than 95 characters and may be truncated");
|
|
15609
|
+
if (tags.description && tags.description.length > 300)
|
|
15610
|
+
warnings.push("og:description is longer than 300 characters and may be truncated");
|
|
15611
|
+
const standardTypes = ["website", "article", "video.movie", "video.episode", "music.song", "music.album", "book", "profile"];
|
|
15612
|
+
if (tags.type && !standardTypes.includes(tags.type))
|
|
15613
|
+
warnings.push(`og:type "${tags.type}" is not a standard type`);
|
|
15614
|
+
return { warnings, missing };
|
|
15615
|
+
}
|
|
15616
|
+
function generateMetaHtml2(tags) {
|
|
15617
|
+
const lines = [];
|
|
15618
|
+
const ogMap = {
|
|
15619
|
+
title: "og:title",
|
|
15620
|
+
description: "og:description",
|
|
15621
|
+
image: "og:image",
|
|
15622
|
+
imageAlt: "og:image:alt",
|
|
15623
|
+
imageWidth: "og:image:width",
|
|
15624
|
+
imageHeight: "og:image:height",
|
|
15625
|
+
url: "og:url",
|
|
15626
|
+
type: "og:type",
|
|
15627
|
+
siteName: "og:site_name",
|
|
15628
|
+
locale: "og:locale"
|
|
15629
|
+
};
|
|
15630
|
+
const twitterMap = {
|
|
15631
|
+
card: "twitter:card",
|
|
15632
|
+
site: "twitter:site",
|
|
15633
|
+
creator: "twitter:creator"
|
|
15634
|
+
};
|
|
15635
|
+
for (const [key, property] of Object.entries(ogMap)) {
|
|
15636
|
+
const v = tags[key];
|
|
15637
|
+
if (v) lines.push(`<meta property="${property}" content="${v}" />`);
|
|
15638
|
+
}
|
|
15639
|
+
for (const [key, name] of Object.entries(twitterMap)) {
|
|
15640
|
+
const v = tags[key];
|
|
15641
|
+
if (v) lines.push(`<meta name="${name}" content="${v}" />`);
|
|
15642
|
+
}
|
|
15643
|
+
return lines.join("\n");
|
|
15644
|
+
}
|
|
15645
|
+
|
|
15646
|
+
// src/tools/ai_agent.ts
|
|
15647
|
+
var ai_agent_exports = {};
|
|
15648
|
+
__export(ai_agent_exports, {
|
|
15649
|
+
MODEL_PRICING: () => MODEL_PRICING,
|
|
15650
|
+
chunkText: () => chunkText,
|
|
15651
|
+
compressHtml: () => compressHtml,
|
|
15652
|
+
compressJson: () => compressJson,
|
|
15653
|
+
compressMarkdown: () => compressMarkdown,
|
|
15654
|
+
deduplicateLines: () => deduplicateLines,
|
|
15655
|
+
detectPii: () => detectPii,
|
|
15656
|
+
detectPromptInjection: () => detectPromptInjection,
|
|
15657
|
+
detectSecrets: () => detectSecrets,
|
|
15658
|
+
diffJson: () => diffJson2,
|
|
15659
|
+
estimateCost: () => estimateCost,
|
|
15660
|
+
estimateTokens: () => estimateTokens,
|
|
15661
|
+
expandQuery: () => expandQuery,
|
|
15662
|
+
extractEntities: () => extractEntities,
|
|
15663
|
+
extractJson: () => extractJson,
|
|
15664
|
+
extractKeywords: () => extractKeywords,
|
|
15665
|
+
extractTables: () => extractTables,
|
|
15666
|
+
extractUrls: () => extractUrls,
|
|
15667
|
+
flattenJson: () => flattenJson,
|
|
15668
|
+
mergeJson: () => mergeJson,
|
|
15669
|
+
redactPii: () => redactPii,
|
|
15670
|
+
repairJson: () => repairJson,
|
|
15671
|
+
rerankChunks: () => rerankChunks,
|
|
15672
|
+
sanitizeHtml: () => sanitizeHtml,
|
|
15673
|
+
scoreRelevance: () => scoreRelevance,
|
|
15674
|
+
summarizeForLlm: () => summarizeForLlm,
|
|
15675
|
+
trimToTokens: () => trimToTokens,
|
|
15676
|
+
unflattenJson: () => unflattenJson,
|
|
15677
|
+
validateJsonSchema: () => validateJsonSchema
|
|
15678
|
+
});
|
|
15679
|
+
var MODEL_PRICING = [
|
|
15680
|
+
{ model: "gpt-4o", label: "GPT-4o", inputPer1M: 5, outputPer1M: 15 },
|
|
15681
|
+
{ model: "gpt-4o-mini", label: "GPT-4o mini", inputPer1M: 0.15, outputPer1M: 0.6 },
|
|
15682
|
+
{ model: "gpt-4-turbo", label: "GPT-4 Turbo", inputPer1M: 10, outputPer1M: 30 },
|
|
15683
|
+
{ model: "claude-sonnet-4", label: "Claude Sonnet 4", inputPer1M: 3, outputPer1M: 15 },
|
|
15684
|
+
{ model: "claude-haiku-4", label: "Claude Haiku 4", inputPer1M: 0.8, outputPer1M: 4 },
|
|
15685
|
+
{ model: "claude-opus-4", label: "Claude Opus 4", inputPer1M: 15, outputPer1M: 75 },
|
|
15686
|
+
{ model: "gemini-1.5-flash", label: "Gemini 1.5 Flash", inputPer1M: 0.075, outputPer1M: 0.3 },
|
|
15687
|
+
{ model: "gemini-1.5-pro", label: "Gemini 1.5 Pro", inputPer1M: 3.5, outputPer1M: 10.5 },
|
|
15688
|
+
{ model: "llama-3.1-70b", label: "Llama 3.1 70B", inputPer1M: 0.59, outputPer1M: 0.79 }
|
|
15689
|
+
];
|
|
15690
|
+
function estimateTokens(text) {
|
|
15691
|
+
if (!text) return { tokens: 0, chars: 0, words: 0, method: "approximate" };
|
|
15692
|
+
const chars = text.length;
|
|
15693
|
+
const words = text.trim() === "" ? 0 : text.trim().split(/\s+/).length;
|
|
15694
|
+
const byChars = Math.ceil(chars / 4);
|
|
15695
|
+
const tokens = Math.ceil(byChars * 0.7 + words * 0.3);
|
|
15696
|
+
return { tokens, chars, words, method: "approximate" };
|
|
15697
|
+
}
|
|
15698
|
+
function estimateCost(inputTokens, outputTokens, model) {
|
|
15699
|
+
const outTokens = outputTokens ?? inputTokens;
|
|
15700
|
+
const pricing = model ? MODEL_PRICING.filter((m) => m.model === model) : MODEL_PRICING;
|
|
15701
|
+
const costs = pricing.map((m) => {
|
|
15702
|
+
const inputCost = inputTokens / 1e6 * m.inputPer1M;
|
|
15703
|
+
const outputCost = outTokens / 1e6 * m.outputPer1M;
|
|
15704
|
+
return { model: m.model, label: m.label, inputCost, outputCost, totalCost: inputCost + outputCost };
|
|
15705
|
+
});
|
|
15706
|
+
return {
|
|
15707
|
+
estimate: estimateTokens(""),
|
|
15708
|
+
// placeholder when called directly
|
|
15709
|
+
costs,
|
|
15710
|
+
model
|
|
15711
|
+
};
|
|
15712
|
+
}
|
|
15713
|
+
var PII_PATTERNS = [
|
|
15714
|
+
{
|
|
15715
|
+
type: "email",
|
|
15716
|
+
label: "Email address",
|
|
15717
|
+
re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g,
|
|
15718
|
+
mask: (v) => v.replace(/^(.{2}).*(@.*)$/, "$1***$2")
|
|
15719
|
+
},
|
|
15720
|
+
{
|
|
15721
|
+
type: "phone",
|
|
15722
|
+
label: "Phone number",
|
|
15723
|
+
re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g,
|
|
15724
|
+
mask: (v) => v.replace(/\d(?=\d{4})/g, "*")
|
|
15725
|
+
},
|
|
15726
|
+
{
|
|
15727
|
+
type: "ssn",
|
|
15728
|
+
label: "Social Security Number",
|
|
15729
|
+
re: /\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b/g,
|
|
15730
|
+
mask: (v) => v.replace(/\d/g, "*").replace(/^\*+/, "***")
|
|
15731
|
+
},
|
|
15732
|
+
{
|
|
15733
|
+
type: "credit_card",
|
|
15734
|
+
label: "Credit card number",
|
|
15735
|
+
re: /\b(?:4\d{12}(?:\d{3})?|5[1-5]\d{14}|3[47]\d{13}|6(?:011|5\d{2})\d{12}|(?:2131|1800|35\d{3})\d{11})\b/g,
|
|
15736
|
+
mask: (v) => v.slice(0, 4) + " **** **** " + v.slice(-4)
|
|
15737
|
+
},
|
|
15738
|
+
{
|
|
15739
|
+
type: "ip_address",
|
|
15740
|
+
label: "IP address",
|
|
15741
|
+
re: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g,
|
|
15742
|
+
mask: (v) => v.replace(/\.\d+$/, ".*")
|
|
15743
|
+
},
|
|
15744
|
+
{
|
|
15745
|
+
type: "iban",
|
|
15746
|
+
label: "IBAN",
|
|
15747
|
+
re: /\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}(?:[A-Z0-9]?){0,16}\b/g,
|
|
15748
|
+
mask: (v) => v.slice(0, 4) + "****" + v.slice(-4)
|
|
15749
|
+
},
|
|
15750
|
+
{
|
|
15751
|
+
type: "crypto_wallet",
|
|
15752
|
+
label: "Crypto wallet address",
|
|
15753
|
+
re: /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-z0-9]{39,59})\b/g,
|
|
15754
|
+
mask: (v) => v.slice(0, 6) + "..." + v.slice(-4)
|
|
15755
|
+
}
|
|
15756
|
+
];
|
|
15757
|
+
function detectPii(text) {
|
|
15758
|
+
const findings = [];
|
|
15759
|
+
for (const p of PII_PATTERNS) {
|
|
15760
|
+
p.re.lastIndex = 0;
|
|
15761
|
+
let m;
|
|
15762
|
+
while ((m = p.re.exec(text)) !== null) {
|
|
15763
|
+
findings.push({
|
|
15764
|
+
type: p.type,
|
|
15765
|
+
label: p.label,
|
|
15766
|
+
value: m[0],
|
|
15767
|
+
masked: p.mask(m[0]),
|
|
15768
|
+
start: m.index,
|
|
15769
|
+
end: m.index + m[0].length
|
|
15770
|
+
});
|
|
15771
|
+
}
|
|
15772
|
+
}
|
|
15773
|
+
findings.sort((a, b) => a.start - b.start);
|
|
15774
|
+
return { found: findings.length > 0, count: findings.length, findings };
|
|
15775
|
+
}
|
|
15776
|
+
function redactPii(text, replacement = "[REDACTED]") {
|
|
15777
|
+
const result = detectPii(text);
|
|
15778
|
+
let redacted = text;
|
|
15779
|
+
for (const f of [...result.findings].reverse()) {
|
|
15780
|
+
redacted = redacted.slice(0, f.start) + replacement + redacted.slice(f.end);
|
|
15781
|
+
}
|
|
15782
|
+
return { ...result, redacted };
|
|
15783
|
+
}
|
|
15784
|
+
var maskSecret = (v) => v.slice(0, 6) + "*".repeat(Math.max(0, v.length - 10)) + v.slice(-4);
|
|
15785
|
+
var SECRET_PATTERNS = [
|
|
15786
|
+
{ type: "openai_key", label: "OpenAI API key", re: /\bsk-[a-zA-Z0-9]{20,}\b/g, risk: "critical" },
|
|
15787
|
+
{ type: "anthropic_key", label: "Anthropic API key", re: /\bsk-ant-[a-zA-Z0-9\-]{20,}\b/g, risk: "critical" },
|
|
15788
|
+
{ type: "aws_key", label: "AWS access key", re: /\bAKIA[0-9A-Z]{16}\b/g, risk: "critical" },
|
|
15789
|
+
{ type: "github_token", label: "GitHub token", re: /\b(?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,}\b/g, risk: "critical" },
|
|
15790
|
+
{ type: "github_token", label: "GitHub fine-grained PAT", re: /\bgithub_pat_[a-zA-Z0-9_]{59,}\b/g, risk: "critical" },
|
|
15791
|
+
{ type: "stripe_key", label: "Stripe key", re: /\b(?:sk|pk)_(?:live|test)_[a-zA-Z0-9]{24,}\b/g, risk: "critical" },
|
|
15792
|
+
{ type: "google_api_key", label: "Google API key", re: /\bAIza[0-9A-Za-z\-_]{35}\b/g, risk: "high" },
|
|
15793
|
+
{ type: "slack_token", label: "Slack token", re: /\bxox[boas]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,}\b/g, risk: "high" },
|
|
15794
|
+
{ type: "sendgrid_key", label: "SendGrid API key", re: /\bSG\.[a-zA-Z0-9\-_]{22}\.[a-zA-Z0-9\-_]{43}\b/g, risk: "high" },
|
|
15795
|
+
{ type: "npm_token", label: "npm token", re: /\bnpm_[a-zA-Z0-9]{36}\b/g, risk: "high" },
|
|
15796
|
+
{ type: "jwt_token", label: "JWT token", re: /\beyJ[a-zA-Z0-9_\-]+\.eyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b/g, risk: "medium" },
|
|
15797
|
+
{ type: "private_key", label: "Private key block", re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g, risk: "critical" },
|
|
15798
|
+
{ type: "generic_api_key", label: "Generic API key pattern", re: /\b(?:api[_-]?key|apikey|api[_-]?secret|access[_-]?token)\s*[:=]\s*['"]?[a-zA-Z0-9\-_]{20,}['"]?/gi, risk: "low" }
|
|
15799
|
+
];
|
|
15800
|
+
function getLineCol(text, index) {
|
|
15801
|
+
const before = text.slice(0, index);
|
|
15802
|
+
const line = (before.match(/\n/g) || []).length + 1;
|
|
15803
|
+
const column = index - before.lastIndexOf("\n");
|
|
15804
|
+
return { line, column };
|
|
15805
|
+
}
|
|
15806
|
+
function detectSecrets(text) {
|
|
15807
|
+
const findings = [];
|
|
15808
|
+
let maxRisk = "none";
|
|
15809
|
+
const riskOrder = { none: 0, low: 1, medium: 2, high: 3, critical: 4 };
|
|
15810
|
+
for (const p of SECRET_PATTERNS) {
|
|
15811
|
+
p.re.lastIndex = 0;
|
|
15812
|
+
let m;
|
|
15813
|
+
while ((m = p.re.exec(text)) !== null) {
|
|
15814
|
+
const { line, column } = getLineCol(text, m.index);
|
|
15815
|
+
findings.push({ type: p.type, label: p.label, value: m[0], masked: maskSecret(m[0]), line, column });
|
|
15816
|
+
if (riskOrder[p.risk] > riskOrder[maxRisk]) maxRisk = p.risk;
|
|
15817
|
+
}
|
|
15818
|
+
}
|
|
15819
|
+
return { found: findings.length > 0, count: findings.length, findings, riskLevel: maxRisk };
|
|
15820
|
+
}
|
|
15821
|
+
var INJECTION_PATTERNS = [
|
|
15822
|
+
{ pattern: "Ignore previous instructions", re: /ignore\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+instructions/gi, severity: "high", weight: 0.9 },
|
|
15823
|
+
{ pattern: "Disregard instructions", re: /disregard\s+(?:all\s+)?(?:previous|prior|above|your)\s+instructions/gi, severity: "high", weight: 0.9 },
|
|
15824
|
+
{ pattern: "New instructions override", re: /(?:new|actual|real)\s+instructions?[::]/gi, severity: "high", weight: 0.8 },
|
|
15825
|
+
{ pattern: "System prompt reveal", re: /(?:reveal|show|print|output|tell me)\s+(?:your\s+)?(?:system\s+prompt|instructions|prompt)/gi, severity: "high", weight: 0.85 },
|
|
15826
|
+
{ pattern: "Role confusion (act as)", re: /\bact\s+as\s+(?:a\s+)?(?:different|new|another|an?\s+ai|dan|jailbreak)/gi, severity: "high", weight: 0.8 },
|
|
15827
|
+
{ pattern: "DAN jailbreak", re: /\bDAN\b|do\s+anything\s+now/gi, severity: "high", weight: 0.95 },
|
|
15828
|
+
{ pattern: "Jailbreak keyword", re: /\b(?:jailbreak|jailbroken|unrestricted\s+mode|developer\s+mode)\b/gi, severity: "high", weight: 0.9 },
|
|
15829
|
+
{ pattern: "System role injection", re: /^system\s*[::]/gim, severity: "high", weight: 0.85 },
|
|
15830
|
+
{ pattern: "Assistant role injection", re: /^assistant\s*[::]/gim, severity: "high", weight: 0.85 },
|
|
15831
|
+
{ pattern: "Prompt injection tag", re: /<(?:system|assistant|user|prompt|instruction)[^>]*>/gi, severity: "medium", weight: 0.7 },
|
|
15832
|
+
{ pattern: "Forget instructions", re: /\b(?:forget|discard|drop)\s+(?:all\s+)?(?:previous\s+)?instructions/gi, severity: "high", weight: 0.85 },
|
|
15833
|
+
{ pattern: "Override safety", re: /(?:override|bypass|circumvent|disable)\s+(?:your\s+)?(?:safety|filters?|restrictions?|guidelines?)/gi, severity: "high", weight: 0.9 },
|
|
15834
|
+
{ pattern: "Hypothetical framing", re: /(?:hypothetically|in\s+a\s+fictional|pretend\s+that|imagine\s+you\s+are)\s+.*(?:no\s+restrictions?|unrestricted)/gi, severity: "medium", weight: 0.6 },
|
|
15835
|
+
{ pattern: "Translate to bypass", re: /translate.*(?:ignore|disregard|bypass)/gi, severity: "medium", weight: 0.5 },
|
|
15836
|
+
{ pattern: "Markdown injection", re: /\[.*\]\(javascript:/gi, severity: "low", weight: 0.4 }
|
|
15837
|
+
];
|
|
15838
|
+
function detectPromptInjection(text) {
|
|
15839
|
+
const findings = [];
|
|
15840
|
+
let totalWeight = 0;
|
|
15841
|
+
for (const p of INJECTION_PATTERNS) {
|
|
15842
|
+
p.re.lastIndex = 0;
|
|
15843
|
+
let m;
|
|
15844
|
+
while ((m = p.re.exec(text)) !== null) {
|
|
15845
|
+
if (!findings.find((f) => f.pattern === p.pattern)) {
|
|
15846
|
+
findings.push({ pattern: p.pattern, matched: m[0], severity: p.severity });
|
|
15847
|
+
totalWeight = Math.min(1, totalWeight + p.weight * (1 - totalWeight));
|
|
15848
|
+
}
|
|
15849
|
+
}
|
|
15850
|
+
}
|
|
15851
|
+
const score = Math.round(totalWeight * 100) / 100;
|
|
15852
|
+
const severity = score === 0 ? "none" : score < 0.4 ? "low" : score < 0.7 ? "medium" : "high";
|
|
15853
|
+
return { isInjection: score > 0.3, score, severity, findings };
|
|
15854
|
+
}
|
|
15855
|
+
function repairJson(input) {
|
|
15856
|
+
const original = input;
|
|
15857
|
+
const fixes = [];
|
|
15858
|
+
let s = input.trim();
|
|
15859
|
+
if (!s) return { ok: false, repaired: "", original, fixes: [], error: "Empty input" };
|
|
15860
|
+
try {
|
|
15861
|
+
JSON.parse(s);
|
|
15862
|
+
return { ok: true, repaired: s, original, fixes: [] };
|
|
15863
|
+
} catch {
|
|
15864
|
+
}
|
|
15865
|
+
const withoutLineComments = s.replace(/\/\/[^\n]*/g, "");
|
|
15866
|
+
const withoutBlockComments = withoutLineComments.replace(/\/\*[\s\S]*?\*\//g, "");
|
|
15867
|
+
if (withoutBlockComments !== s) {
|
|
15868
|
+
s = withoutBlockComments;
|
|
15869
|
+
fixes.push("Removed JS comments");
|
|
15870
|
+
}
|
|
15871
|
+
const singleToDouble = s.replace(/'([^'\\]*(\\.[^'\\]*)*)'/g, (_, inner) => `"${inner.replace(/"/g, '\\"')}"`);
|
|
15872
|
+
if (singleToDouble !== s) {
|
|
15873
|
+
s = singleToDouble;
|
|
15874
|
+
fixes.push("Replaced single quotes with double quotes");
|
|
15875
|
+
}
|
|
15876
|
+
const quotedKeys = s.replace(/([{,]\s*)([a-zA-Z_$][a-zA-Z0-9_$]*)\s*:/g, '$1"$2":');
|
|
15877
|
+
if (quotedKeys !== s) {
|
|
15878
|
+
s = quotedKeys;
|
|
15879
|
+
fixes.push("Quoted unquoted object keys");
|
|
15880
|
+
}
|
|
15881
|
+
const noTrailingCommas = s.replace(/,(\s*[}\]])/g, "$1");
|
|
15882
|
+
if (noTrailingCommas !== s) {
|
|
15883
|
+
s = noTrailingCommas;
|
|
15884
|
+
fixes.push("Removed trailing commas");
|
|
15885
|
+
}
|
|
15886
|
+
const noUndefined = s.replace(/\bundefined\b/g, "null");
|
|
15887
|
+
if (noUndefined !== s) {
|
|
15888
|
+
s = noUndefined;
|
|
15889
|
+
fixes.push("Replaced undefined with null");
|
|
15890
|
+
}
|
|
15891
|
+
const noNaN = s.replace(/\bNaN\b/g, "null").replace(/\b(?:Infinity|-Infinity)\b/g, "null");
|
|
15892
|
+
if (noNaN !== s) {
|
|
15893
|
+
s = noNaN;
|
|
15894
|
+
fixes.push("Replaced NaN/Infinity with null");
|
|
15895
|
+
}
|
|
15896
|
+
const opens = [];
|
|
15897
|
+
let inStr = false;
|
|
15898
|
+
let escape = false;
|
|
15899
|
+
for (const ch of s) {
|
|
15900
|
+
if (escape) {
|
|
15901
|
+
escape = false;
|
|
15902
|
+
continue;
|
|
15903
|
+
}
|
|
15904
|
+
if (ch === "\\" && inStr) {
|
|
15905
|
+
escape = true;
|
|
15906
|
+
continue;
|
|
15907
|
+
}
|
|
15908
|
+
if (ch === '"') {
|
|
15909
|
+
inStr = !inStr;
|
|
15910
|
+
continue;
|
|
15911
|
+
}
|
|
15912
|
+
if (inStr) continue;
|
|
15913
|
+
if (ch === "{") opens.push("}");
|
|
15914
|
+
else if (ch === "[") opens.push("]");
|
|
15915
|
+
else if (ch === "}" || ch === "]") opens.pop();
|
|
15916
|
+
}
|
|
15917
|
+
if (opens.length > 0) {
|
|
15918
|
+
s += opens.reverse().join("");
|
|
15919
|
+
fixes.push(`Closed ${opens.length} unclosed bracket(s)`);
|
|
15920
|
+
}
|
|
15921
|
+
try {
|
|
15922
|
+
JSON.parse(s);
|
|
15923
|
+
return { ok: true, repaired: s, original, fixes };
|
|
15924
|
+
} catch (e) {
|
|
15925
|
+
return { ok: false, repaired: s, original, fixes, error: e.message };
|
|
15926
|
+
}
|
|
15927
|
+
}
|
|
15928
|
+
function _estimateTokens(text) {
|
|
15929
|
+
const chars = text.length;
|
|
15930
|
+
const words = text.trim() === "" ? 0 : text.trim().split(/\s+/).length;
|
|
15931
|
+
return Math.ceil(Math.ceil(chars / 4) * 0.7 + words * 0.3);
|
|
15932
|
+
}
|
|
15933
|
+
function trimToTokens(text, maxTokens, strategy = "end") {
|
|
15934
|
+
const originalTokens = _estimateTokens(text);
|
|
15935
|
+
if (originalTokens <= maxTokens) return { trimmed: text, originalTokens, trimmedTokens: originalTokens, truncated: false, removedChars: 0 };
|
|
15936
|
+
const fit = (s) => _estimateTokens(s) <= maxTokens;
|
|
15937
|
+
if (strategy === "end") {
|
|
15938
|
+
let lo2 = 0, hi2 = text.length;
|
|
15939
|
+
while (lo2 < hi2) {
|
|
15940
|
+
const mid = lo2 + hi2 + 1 >> 1;
|
|
15941
|
+
fit(text.slice(0, mid)) ? lo2 = mid : hi2 = mid - 1;
|
|
15942
|
+
}
|
|
15943
|
+
const trimmed2 = text.slice(0, lo2).trimEnd();
|
|
15944
|
+
return { trimmed: trimmed2, originalTokens, trimmedTokens: _estimateTokens(trimmed2), truncated: true, removedChars: text.length - lo2 };
|
|
15945
|
+
}
|
|
15946
|
+
if (strategy === "start") {
|
|
15947
|
+
let lo2 = 0, hi2 = text.length;
|
|
15948
|
+
while (lo2 < hi2) {
|
|
15949
|
+
const mid = lo2 + hi2 >> 1;
|
|
15950
|
+
fit(text.slice(mid)) ? hi2 = mid : lo2 = mid + 1;
|
|
15951
|
+
}
|
|
15952
|
+
const trimmed2 = text.slice(lo2).trimStart();
|
|
15953
|
+
return { trimmed: trimmed2, originalTokens, trimmedTokens: _estimateTokens(trimmed2), truncated: true, removedChars: lo2 };
|
|
15954
|
+
}
|
|
15955
|
+
let lo = 0, hi = text.length;
|
|
15956
|
+
while (lo < hi) {
|
|
15957
|
+
const mid = lo + hi + 1 >> 1;
|
|
15958
|
+
const h2 = mid >> 1;
|
|
15959
|
+
fit(text.slice(0, h2) + "\n...\n" + text.slice(text.length - h2)) ? lo = mid : hi = mid - 1;
|
|
15960
|
+
}
|
|
15961
|
+
const h = lo >> 1;
|
|
15962
|
+
const trimmed = text.slice(0, h).trimEnd() + "\n...\n" + text.slice(text.length - h).trimStart();
|
|
15963
|
+
return { trimmed, originalTokens, trimmedTokens: _estimateTokens(trimmed), truncated: true, removedChars: text.length - lo };
|
|
15964
|
+
}
|
|
15965
|
+
function chunkText(text, chunkSize = 200, overlap = 20, strategy = "paragraph") {
|
|
15966
|
+
let units;
|
|
15967
|
+
if (strategy === "paragraph") units = text.split(/\n{2,}/).filter((s) => s.trim());
|
|
15968
|
+
else if (strategy === "sentence") units = text.match(/[^.!?]+[.!?]+(\s|$)|[^.!?]+$/g)?.map((s) => s.trim()).filter(Boolean) ?? [text];
|
|
15969
|
+
else {
|
|
15970
|
+
const words = text.split(/\s+/);
|
|
15971
|
+
const us = [];
|
|
15972
|
+
let buf = "";
|
|
15973
|
+
for (const w of words) {
|
|
15974
|
+
if (buf && (buf + " " + w).length > 200) {
|
|
15975
|
+
us.push(buf);
|
|
15976
|
+
buf = w;
|
|
15977
|
+
} else buf = buf ? buf + " " + w : w;
|
|
15978
|
+
}
|
|
15979
|
+
if (buf) us.push(buf);
|
|
15980
|
+
units = us;
|
|
15981
|
+
}
|
|
15982
|
+
const chunks = [];
|
|
15983
|
+
let i = 0, charPos = 0;
|
|
15984
|
+
while (i < units.length) {
|
|
15985
|
+
const buf = [];
|
|
15986
|
+
let tokens = 0, j = i;
|
|
15987
|
+
while (j < units.length && tokens < chunkSize) {
|
|
15988
|
+
buf.push(units[j]);
|
|
15989
|
+
tokens = _estimateTokens(buf.join(strategy === "paragraph" ? "\n\n" : " "));
|
|
15990
|
+
j++;
|
|
15991
|
+
}
|
|
15992
|
+
const ct = buf.join(strategy === "paragraph" ? "\n\n" : " ");
|
|
15993
|
+
const sc = Math.max(0, text.indexOf(buf[0], charPos));
|
|
15994
|
+
chunks.push({ index: chunks.length, text: ct, tokens: _estimateTokens(ct), startChar: sc, endChar: sc + ct.length });
|
|
15995
|
+
const ov = Math.max(1, Math.floor(overlap / Math.max(1, tokens / buf.length)));
|
|
15996
|
+
i = Math.max(i + 1, j - ov);
|
|
15997
|
+
charPos = sc;
|
|
15998
|
+
}
|
|
15999
|
+
return chunks;
|
|
16000
|
+
}
|
|
16001
|
+
function extractUrls(text) {
|
|
16002
|
+
const seen = /* @__PURE__ */ new Set();
|
|
16003
|
+
const results = [];
|
|
16004
|
+
const hrefRe = /href=["']([^"']+)["']/gi;
|
|
16005
|
+
let m;
|
|
16006
|
+
while ((m = hrefRe.exec(text)) !== null) {
|
|
16007
|
+
const url = m[1];
|
|
16008
|
+
if (url.startsWith("http") && !seen.has(url)) {
|
|
16009
|
+
seen.add(url);
|
|
16010
|
+
results.push(parseUrl2(url, m.index));
|
|
16011
|
+
}
|
|
16012
|
+
}
|
|
16013
|
+
const urlRe = /https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_+.~#?&/=]*)/g;
|
|
16014
|
+
while ((m = urlRe.exec(text)) !== null) {
|
|
16015
|
+
if (!seen.has(m[0])) {
|
|
16016
|
+
seen.add(m[0]);
|
|
16017
|
+
results.push(parseUrl2(m[0], m.index));
|
|
16018
|
+
}
|
|
16019
|
+
}
|
|
16020
|
+
results.sort((a, b) => a.position - b.position);
|
|
16021
|
+
return { urls: results, count: results.length };
|
|
16022
|
+
}
|
|
16023
|
+
function parseUrl2(url, position) {
|
|
16024
|
+
try {
|
|
16025
|
+
const u = new URL(url);
|
|
16026
|
+
return { url, protocol: u.protocol.replace(":", ""), domain: u.hostname, path: u.pathname + u.search + u.hash, position };
|
|
16027
|
+
} catch {
|
|
16028
|
+
return { url, protocol: url.startsWith("https") ? "https" : "http", domain: "", path: "", position };
|
|
16029
|
+
}
|
|
16030
|
+
}
|
|
16031
|
+
function sanitizeHtml(html, keepLinks = false) {
|
|
16032
|
+
let s = html;
|
|
16033
|
+
const scripts = s.match(/<script[\s\S]*?<\/script>/gi) ?? [];
|
|
16034
|
+
s = s.replace(/<script[\s\S]*?<\/script>/gi, "");
|
|
16035
|
+
const styles = s.match(/<style[\s\S]*?<\/style>/gi) ?? [];
|
|
16036
|
+
s = s.replace(/<style[\s\S]*?<\/style>/gi, "");
|
|
16037
|
+
s = s.replace(/<!--[\s\S]*?-->/g, "").replace(/\son\w+\s*=\s*["'][^"']*["']/gi, "").replace(/\son\w+\s*=\s*[^\s>]*/gi, "");
|
|
16038
|
+
if (keepLinks) s = s.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, "$2 [$1]");
|
|
16039
|
+
s = s.replace(/<\/?(p|div|h[1-6]|li|tr|td|th|br|hr|blockquote|section|article|header|footer|main|nav)[^>]*>/gi, "\n");
|
|
16040
|
+
const remaining = (s.match(/<[^>]+>/g) ?? []).length;
|
|
16041
|
+
s = s.replace(/<[^>]+>/g, "").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/ /g, " ");
|
|
16042
|
+
s = s.replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
|
|
16043
|
+
return { text: s, removedTags: remaining + scripts.length + styles.length, removedScripts: scripts.length, removedStyles: styles.length, originalLength: html.length, cleanLength: s.length };
|
|
16044
|
+
}
|
|
16045
|
+
function flattenJson(obj, separator = ".", prefix = "") {
|
|
16046
|
+
const result = {};
|
|
16047
|
+
function walk(value, path) {
|
|
16048
|
+
if (value === null || typeof value !== "object") {
|
|
16049
|
+
result[path] = value;
|
|
16050
|
+
return;
|
|
16051
|
+
}
|
|
16052
|
+
if (Array.isArray(value)) {
|
|
16053
|
+
if (value.length === 0) {
|
|
16054
|
+
result[path] = [];
|
|
16055
|
+
return;
|
|
16056
|
+
}
|
|
16057
|
+
value.forEach((item, i) => walk(item, path ? `${path}[${i}]` : `[${i}]`));
|
|
16058
|
+
return;
|
|
16059
|
+
}
|
|
16060
|
+
const entries = Object.entries(value);
|
|
16061
|
+
if (entries.length === 0) {
|
|
16062
|
+
result[path] = {};
|
|
16063
|
+
return;
|
|
16064
|
+
}
|
|
16065
|
+
for (const [k, v] of entries) walk(v, path ? `${path}${separator}${k}` : k);
|
|
16066
|
+
}
|
|
16067
|
+
walk(obj, prefix);
|
|
16068
|
+
return result;
|
|
16069
|
+
}
|
|
16070
|
+
function unflattenJson(flat, separator = ".") {
|
|
16071
|
+
const result = {};
|
|
16072
|
+
for (const [flatKey, value] of Object.entries(flat)) {
|
|
16073
|
+
const parts = flatKey.split(separator).flatMap((p) => {
|
|
16074
|
+
const m = p.match(/^([^\[]*)((?:\[\d+\])*)$/);
|
|
16075
|
+
if (!m) return [p];
|
|
16076
|
+
const ps = [];
|
|
16077
|
+
if (m[1]) ps.push(m[1]);
|
|
16078
|
+
for (const idx of m[2].matchAll(/\[(\d+)\]/g)) ps.push(idx[1]);
|
|
16079
|
+
return ps;
|
|
16080
|
+
});
|
|
16081
|
+
let cur = result;
|
|
16082
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
16083
|
+
const key = parts[i], nk = parts[i + 1];
|
|
16084
|
+
if (!(key in cur)) cur[key] = /^\d+$/.test(nk) ? [] : {};
|
|
16085
|
+
cur = cur[key];
|
|
16086
|
+
}
|
|
16087
|
+
cur[parts[parts.length - 1]] = value;
|
|
16088
|
+
}
|
|
16089
|
+
return result;
|
|
16090
|
+
}
|
|
16091
|
+
function _isObj(v) {
|
|
16092
|
+
return v !== null && typeof v === "object" && !Array.isArray(v);
|
|
16093
|
+
}
|
|
16094
|
+
function _deepMerge(base, override, strategy, path, conflicts) {
|
|
16095
|
+
if (_isObj(base) && _isObj(override)) {
|
|
16096
|
+
const result = { ...base };
|
|
16097
|
+
for (const [key, val] of Object.entries(override)) {
|
|
16098
|
+
const np = path ? `${path}.${key}` : key;
|
|
16099
|
+
result[key] = key in result ? _deepMerge(result[key], val, strategy, np, conflicts) : val;
|
|
16100
|
+
}
|
|
16101
|
+
return result;
|
|
16102
|
+
}
|
|
16103
|
+
if (Array.isArray(base) && Array.isArray(override)) {
|
|
16104
|
+
if (strategy !== "deep") return override;
|
|
16105
|
+
return [...base, ...override];
|
|
16106
|
+
}
|
|
16107
|
+
if (base !== void 0 && override !== void 0 && JSON.stringify(base) !== JSON.stringify(override)) conflicts.push({ path, base, override });
|
|
16108
|
+
return override;
|
|
16109
|
+
}
|
|
16110
|
+
function mergeJson(base, override, strategy = "deep") {
|
|
16111
|
+
const conflicts = [];
|
|
16112
|
+
return { merged: _deepMerge(base, override, strategy, "", conflicts), conflicts };
|
|
16113
|
+
}
|
|
16114
|
+
function extractJson(text) {
|
|
16115
|
+
const blocks = [];
|
|
16116
|
+
let i = 0;
|
|
16117
|
+
const cbRe = /```(?:json|JSON)?\s*\n?([\s\S]*?)```/g;
|
|
16118
|
+
let m;
|
|
16119
|
+
while ((m = cbRe.exec(text)) !== null) {
|
|
16120
|
+
try {
|
|
16121
|
+
const raw = m[1].trim();
|
|
16122
|
+
const parsed = JSON.parse(raw);
|
|
16123
|
+
blocks.push({ index: 0, raw, parsed, type: Array.isArray(parsed) ? "array" : typeof parsed, source: "code-block", startPos: m.index });
|
|
16124
|
+
} catch {
|
|
16125
|
+
}
|
|
16126
|
+
}
|
|
16127
|
+
const cbRanges = [...text.matchAll(/```[\s\S]*?```/g)].map((m2) => [m2.index, m2.index + m2[0].length]);
|
|
16128
|
+
const inCb = (pos) => cbRanges.some(([s, e]) => pos >= s && pos < e);
|
|
16129
|
+
while (i < text.length) {
|
|
16130
|
+
const ch = text[i];
|
|
16131
|
+
if ((ch === "{" || ch === "[") && !inCb(i)) {
|
|
16132
|
+
let depth = 1, j = i + 1, inStr = false;
|
|
16133
|
+
while (j < text.length && depth > 0) {
|
|
16134
|
+
const c = text[j];
|
|
16135
|
+
if (c === '"' && text[j - 1] !== "\\") inStr = !inStr;
|
|
16136
|
+
if (!inStr) {
|
|
16137
|
+
if (c === "{" || c === "[") depth++;
|
|
16138
|
+
else if (c === "}" || c === "]") depth--;
|
|
16139
|
+
}
|
|
16140
|
+
j++;
|
|
16141
|
+
}
|
|
16142
|
+
if (depth === 0) {
|
|
16143
|
+
const raw = text.slice(i, j);
|
|
16144
|
+
try {
|
|
16145
|
+
const parsed = JSON.parse(raw);
|
|
16146
|
+
if (!blocks.some((b) => b.raw.trim() === raw.trim()) && raw.length > 2) blocks.push({ index: 0, raw, parsed, type: Array.isArray(parsed) ? "array" : typeof parsed, source: "inline", startPos: i });
|
|
16147
|
+
i = j;
|
|
16148
|
+
continue;
|
|
16149
|
+
} catch {
|
|
16150
|
+
}
|
|
16151
|
+
}
|
|
16152
|
+
}
|
|
16153
|
+
i++;
|
|
16154
|
+
}
|
|
16155
|
+
blocks.sort((a, b) => a.startPos - b.startPos).forEach((b, idx) => {
|
|
16156
|
+
b.index = idx;
|
|
16157
|
+
});
|
|
16158
|
+
return { blocks, count: blocks.length };
|
|
16159
|
+
}
|
|
16160
|
+
function deduplicateLines(text, strategy = "exact") {
|
|
16161
|
+
const raw = text.split("\n");
|
|
16162
|
+
const norm = (s) => strategy === "exact" ? s : strategy === "case-insensitive" ? s.toLowerCase() : strategy === "trimmed" ? s.trim() : s.trim().replace(/\s+/g, " ").toLowerCase();
|
|
16163
|
+
const seen = /* @__PURE__ */ new Map();
|
|
16164
|
+
const result = [];
|
|
16165
|
+
for (let i = 0; i < raw.length; i++) {
|
|
16166
|
+
const key = norm(raw[i]);
|
|
16167
|
+
if (!seen.has(key)) {
|
|
16168
|
+
seen.set(key, { count: 1, firstLine: i + 1, original: raw[i] });
|
|
16169
|
+
result.push(raw[i]);
|
|
16170
|
+
} else seen.get(key).count++;
|
|
16171
|
+
}
|
|
16172
|
+
const duplicates = [...seen.entries()].filter(([, v]) => v.count > 1).map(([, v]) => ({ line: v.original, count: v.count, firstLine: v.firstLine })).sort((a, b) => b.count - a.count);
|
|
16173
|
+
return { lines: result, originalCount: raw.length, uniqueCount: result.length, removedCount: raw.length - result.length, duplicates };
|
|
16174
|
+
}
|
|
16175
|
+
var _STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "is", "was", "are", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "must", "shall", "not", "no", "nor", "so", "yet", "both", "either", "neither", "as", "if", "then", "than", "that", "this", "these", "those", "it", "its", "i", "we", "you", "he", "she", "they", "their", "there", "here", "what", "which", "who", "when", "where", "how", "why", "all", "any", "each", "every", "more", "most", "other", "some", "such", "into", "over", "also", "just", "about", "up", "out", "can", "get", "only", "new"]);
|
|
16176
|
+
function extractKeywords(text, maxKeywords = 20, minWordLen = 3) {
|
|
16177
|
+
const words = text.toLowerCase().match(/\b[a-z]{3,}\b/g) ?? [];
|
|
16178
|
+
const freq = /* @__PURE__ */ new Map();
|
|
16179
|
+
words.forEach((w, pos) => {
|
|
16180
|
+
if (w.length >= minWordLen && !_STOP.has(w)) {
|
|
16181
|
+
if (!freq.has(w)) freq.set(w, { count: 0, positions: [] });
|
|
16182
|
+
freq.get(w).count++;
|
|
16183
|
+
freq.get(w).positions.push(pos);
|
|
16184
|
+
}
|
|
16185
|
+
});
|
|
16186
|
+
const maxFreq = Math.max(...[...freq.values()].map((v) => v.count), 1);
|
|
16187
|
+
const sentences = text.split(/[.!?]+/).filter(Boolean);
|
|
16188
|
+
const keywords = [...freq.entries()].map(([word, { count, positions }]) => {
|
|
16189
|
+
const tf = count / words.length;
|
|
16190
|
+
const df = sentences.filter((s) => s.toLowerCase().includes(word)).length || 1;
|
|
16191
|
+
const idf = Math.log((sentences.length + 1) / df);
|
|
16192
|
+
return { word, count, score: Math.round(tf * idf * (count / maxFreq) * 1e3) / 100, positions };
|
|
16193
|
+
}).sort((a, b) => b.score - a.score || b.count - a.count).slice(0, maxKeywords);
|
|
16194
|
+
return { keywords, totalWords: words.length, uniqueWords: freq.size };
|
|
16195
|
+
}
|
|
16196
|
+
function _validate(data, schema, path, errors) {
|
|
16197
|
+
const typeOf = (v) => v === null ? "null" : Array.isArray(v) ? "array" : typeof v;
|
|
16198
|
+
if (schema.type) {
|
|
16199
|
+
const types = Array.isArray(schema.type) ? schema.type : [schema.type];
|
|
16200
|
+
if (!types.some((t) => t === typeOf(data) || t === "integer" && typeOf(data) === "number" && Number.isInteger(data))) errors.push({ path, message: `Expected ${types.join("|")}, got ${typeOf(data)}`, value: data });
|
|
16201
|
+
}
|
|
16202
|
+
if ("enum" in schema && Array.isArray(schema.enum) && !schema.enum.some((e) => JSON.stringify(e) === JSON.stringify(data))) errors.push({ path, message: `Must be one of: ${schema.enum.map((e) => JSON.stringify(e)).join(", ")}` });
|
|
16203
|
+
if (typeof data === "string") {
|
|
16204
|
+
if (typeof schema.minLength === "number" && data.length < schema.minLength) errors.push({ path, message: `Too short (min ${schema.minLength})` });
|
|
16205
|
+
if (typeof schema.maxLength === "number" && data.length > schema.maxLength) errors.push({ path, message: `Too long (max ${schema.maxLength})` });
|
|
16206
|
+
if (typeof schema.pattern === "string") {
|
|
16207
|
+
try {
|
|
16208
|
+
if (!new RegExp(schema.pattern).test(data)) errors.push({ path, message: `Does not match pattern ${schema.pattern}` });
|
|
16209
|
+
} catch {
|
|
16210
|
+
}
|
|
16211
|
+
}
|
|
16212
|
+
}
|
|
16213
|
+
if (typeof data === "number") {
|
|
16214
|
+
if (typeof schema.minimum === "number" && data < schema.minimum) errors.push({ path, message: `Must be >= ${schema.minimum}` });
|
|
16215
|
+
if (typeof schema.maximum === "number" && data > schema.maximum) errors.push({ path, message: `Must be <= ${schema.maximum}` });
|
|
16216
|
+
}
|
|
16217
|
+
if (Array.isArray(data)) {
|
|
16218
|
+
if (typeof schema.minItems === "number" && data.length < schema.minItems) errors.push({ path, message: `Too few items (min ${schema.minItems})` });
|
|
16219
|
+
if (schema.items && typeof schema.items === "object") data.forEach((item, i) => _validate(item, schema.items, `${path}[${i}]`, errors));
|
|
16220
|
+
}
|
|
16221
|
+
if (data !== null && typeof data === "object" && !Array.isArray(data)) {
|
|
16222
|
+
const obj = data;
|
|
16223
|
+
if (Array.isArray(schema.required)) {
|
|
16224
|
+
for (const k of schema.required) if (!(k in obj)) errors.push({ path: path ? `${path}.${k}` : k, message: "Required property missing" });
|
|
16225
|
+
}
|
|
16226
|
+
if (schema.properties) {
|
|
16227
|
+
for (const [k, sub] of Object.entries(schema.properties)) if (k in obj) _validate(obj[k], sub, path ? `${path}.${k}` : k, errors);
|
|
16228
|
+
}
|
|
16229
|
+
}
|
|
16230
|
+
if (Array.isArray(schema.allOf)) for (const sub of schema.allOf) _validate(data, sub, path, errors);
|
|
16231
|
+
if (Array.isArray(schema.anyOf)) {
|
|
16232
|
+
const any = schema.anyOf.map((sub) => {
|
|
16233
|
+
const e = [];
|
|
16234
|
+
_validate(data, sub, path, e);
|
|
16235
|
+
return e;
|
|
16236
|
+
});
|
|
16237
|
+
if (any.every((e) => e.length > 0)) errors.push({ path, message: "Must match at least one schema in anyOf" });
|
|
16238
|
+
}
|
|
16239
|
+
}
|
|
16240
|
+
function validateJsonSchema(data, schema) {
|
|
16241
|
+
const errors = [];
|
|
16242
|
+
_validate(data, schema, "", errors);
|
|
16243
|
+
return { valid: errors.length === 0, errors };
|
|
16244
|
+
}
|
|
16245
|
+
function _isObjD(v) {
|
|
16246
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
16247
|
+
}
|
|
16248
|
+
function _deepEq(a, b) {
|
|
16249
|
+
if (a === b) return true;
|
|
16250
|
+
if (typeof a !== typeof b) return false;
|
|
16251
|
+
if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((v, i) => _deepEq(v, b[i]));
|
|
16252
|
+
if (_isObjD(a) && _isObjD(b)) {
|
|
16253
|
+
const ka = Object.keys(a), kb = Object.keys(b);
|
|
16254
|
+
return ka.length === kb.length && ka.every((k) => _deepEq(a[k], b[k]));
|
|
16255
|
+
}
|
|
16256
|
+
return false;
|
|
16257
|
+
}
|
|
16258
|
+
function _walkDiff(a, b, path, entries) {
|
|
16259
|
+
if (_isObjD(a) && _isObjD(b)) {
|
|
16260
|
+
for (const k of /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)])) {
|
|
16261
|
+
const p = path ? `${path}.${k}` : k;
|
|
16262
|
+
if (!(k in a)) entries.push({ path: p, op: "added", newValue: b[k] });
|
|
16263
|
+
else if (!(k in b)) entries.push({ path: p, op: "removed", oldValue: a[k] });
|
|
16264
|
+
else _walkDiff(a[k], b[k], p, entries);
|
|
16265
|
+
}
|
|
16266
|
+
} else if (Array.isArray(a) && Array.isArray(b)) {
|
|
16267
|
+
const len = Math.max(a.length, b.length);
|
|
16268
|
+
for (let i = 0; i < len; i++) {
|
|
16269
|
+
const p = `${path}[${i}]`;
|
|
16270
|
+
if (i >= a.length) entries.push({ path: p, op: "added", newValue: b[i] });
|
|
16271
|
+
else if (i >= b.length) entries.push({ path: p, op: "removed", oldValue: a[i] });
|
|
16272
|
+
else _walkDiff(a[i], b[i], p, entries);
|
|
16273
|
+
}
|
|
16274
|
+
} else {
|
|
16275
|
+
if (_deepEq(a, b)) entries.push({ path, op: "unchanged", oldValue: a, newValue: b });
|
|
16276
|
+
else entries.push({ path, op: "changed", oldValue: a, newValue: b });
|
|
16277
|
+
}
|
|
16278
|
+
}
|
|
16279
|
+
function diffJson2(a, b) {
|
|
16280
|
+
const entries = [];
|
|
16281
|
+
_walkDiff(a, b, "", entries);
|
|
16282
|
+
return { entries, added: entries.filter((e) => e.op === "added").length, removed: entries.filter((e) => e.op === "removed").length, changed: entries.filter((e) => e.op === "changed").length, unchanged: entries.filter((e) => e.op === "unchanged").length, totalPaths: entries.length };
|
|
16283
|
+
}
|
|
16284
|
+
function _stripTags(h) {
|
|
16285
|
+
return h.replace(/<[^>]+>/g, "").replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/ /g, " ").replace(/"/g, '"').trim();
|
|
16286
|
+
}
|
|
16287
|
+
function _cells(rowHtml, tag) {
|
|
16288
|
+
const c = [], re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "gi");
|
|
16289
|
+
let m;
|
|
16290
|
+
while ((m = re.exec(rowHtml)) !== null) c.push(_stripTags(m[1]));
|
|
16291
|
+
return c;
|
|
16292
|
+
}
|
|
16293
|
+
function extractTables(html) {
|
|
16294
|
+
const tables = [];
|
|
16295
|
+
const tr = /<table[^>]*>([\s\S]*?)<\/table>/gi;
|
|
16296
|
+
let tm;
|
|
16297
|
+
let idx = 0;
|
|
16298
|
+
while ((tm = tr.exec(html)) !== null) {
|
|
16299
|
+
const body = tm[1], capM = body.match(/<caption[^>]*>([\s\S]*?)<\/caption>/i), caption = capM ? _stripTags(capM[1]) : "";
|
|
16300
|
+
let headers = [];
|
|
16301
|
+
const theadM = body.match(/<thead[^>]*>([\s\S]*?)<\/thead>/i);
|
|
16302
|
+
if (theadM) {
|
|
16303
|
+
const r = theadM[1].match(/<tr[^>]*>([\s\S]*?)<\/tr>/i);
|
|
16304
|
+
if (r) headers = _cells(r[1], "th").concat(_cells(r[1], "td"));
|
|
16305
|
+
}
|
|
16306
|
+
if (!headers.length) {
|
|
16307
|
+
const f = body.match(/<tr[^>]*>([\s\S]*?)<\/tr>/i);
|
|
16308
|
+
if (f) headers = _cells(f[1], "th");
|
|
16309
|
+
}
|
|
16310
|
+
const rows = [], tbM = body.match(/<tbody[^>]*>([\s\S]*?)<\/tbody>/i), src = tbM ? tbM[1] : body;
|
|
16311
|
+
const rr = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
|
|
16312
|
+
let rm;
|
|
16313
|
+
while ((rm = rr.exec(src)) !== null) {
|
|
16314
|
+
const c = _cells(rm[1], "td");
|
|
16315
|
+
if (c.length) rows.push(c);
|
|
16316
|
+
}
|
|
16317
|
+
tables.push({ index: idx++, caption, headers, rows, rowCount: rows.length, colCount: Math.max(headers.length, ...rows.map((r) => r.length)) });
|
|
16318
|
+
}
|
|
16319
|
+
return { tables, count: tables.length };
|
|
16320
|
+
}
|
|
16321
|
+
var _ENT_PATTERNS = [
|
|
16322
|
+
{ type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
|
|
16323
|
+
{ type: "phone", re: /(?<!\d)(\+?1[\s.-]?)?(\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4})(?!\d)/g },
|
|
16324
|
+
{ type: "date", re: /\b(\d{4}[-\/]\d{2}[-\/]\d{2}|\d{2}[-\/]\d{2}[-\/]\d{4}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})\b/gi },
|
|
16325
|
+
{ type: "ip", re: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g },
|
|
16326
|
+
{ type: "creditCard", re: /\b(?:\d{4}[\s\-]?){3}\d{4}\b/g },
|
|
16327
|
+
{ type: "ssn", re: /\b\d{3}-\d{2}-\d{4}\b/g },
|
|
16328
|
+
{ type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z0-9]?){0,16}\b/g },
|
|
16329
|
+
{ type: "hashtag", re: /#[a-zA-Z]\w*/g },
|
|
16330
|
+
{ type: "mention", re: /@[a-zA-Z]\w*/g },
|
|
16331
|
+
{ type: "currency", re: /(?:USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|INR)\s*[\d,]+(?:\.\d{1,2})?|[\$£€¥₹]\s*[\d,]+(?:\.\d{1,2})?/g }
|
|
16332
|
+
];
|
|
16333
|
+
function extractEntities(text) {
|
|
16334
|
+
const entities = [], seen = /* @__PURE__ */ new Set();
|
|
16335
|
+
for (const { type, re } of _ENT_PATTERNS) {
|
|
16336
|
+
re.lastIndex = 0;
|
|
16337
|
+
let m;
|
|
16338
|
+
while ((m = re.exec(text)) !== null) {
|
|
16339
|
+
const k = `${type}:${m[0]}`;
|
|
16340
|
+
if (!seen.has(k)) {
|
|
16341
|
+
seen.add(k);
|
|
16342
|
+
entities.push({ type, value: m[0].trim(), position: m.index });
|
|
16343
|
+
}
|
|
16344
|
+
}
|
|
16345
|
+
}
|
|
16346
|
+
entities.sort((a, b) => a.position - b.position);
|
|
16347
|
+
const byType = {};
|
|
16348
|
+
for (const e of entities) {
|
|
16349
|
+
if (!byType[e.type]) byType[e.type] = [];
|
|
16350
|
+
byType[e.type].push(e);
|
|
16351
|
+
}
|
|
16352
|
+
return { entities, byType, count: entities.length };
|
|
16353
|
+
}
|
|
16354
|
+
function compressHtml(html, opts = {}) {
|
|
16355
|
+
const { removeComments = true, removeScripts = true, removeStyles = true, collapseWhitespace = true } = opts;
|
|
16356
|
+
let s = html;
|
|
16357
|
+
if (removeScripts) s = s.replace(/<script[\s\S]*?<\/script>/gi, "");
|
|
16358
|
+
if (removeStyles) s = s.replace(/<style[\s\S]*?<\/style>/gi, "");
|
|
16359
|
+
if (removeComments) s = s.replace(/<!--[\s\S]*?-->/g, "");
|
|
16360
|
+
s = s.replace(/\s+data-[a-z\-]+="[^"]*"/gi, "").replace(/\s+class="[^"]*"/gi, "").replace(/\s+style="[^"]*"/gi, "").replace(/\s+id="[^"]*"/gi, "");
|
|
16361
|
+
if (collapseWhitespace) {
|
|
16362
|
+
s = s.replace(/>\s+</g, "><").replace(/\s{2,}/g, " ").trim();
|
|
16363
|
+
}
|
|
16364
|
+
const enc = (t) => Buffer.byteLength(t, "utf8");
|
|
16365
|
+
const originalBytes = enc(html), compressedBytes = enc(s), originalTokens = Math.ceil(html.length / 4), compressedTokens = Math.ceil(s.length / 4);
|
|
16366
|
+
return { compressed: s, originalBytes, compressedBytes, originalTokens, compressedTokens, savings: originalBytes - compressedBytes, savingsPct: originalBytes > 0 ? Math.round((originalBytes - compressedBytes) / originalBytes * 100) : 0 };
|
|
16367
|
+
}
|
|
16368
|
+
function compressMarkdown(md, opts = {}) {
|
|
16369
|
+
const { collapseBlankLines = true, removeComments = true, stripFrontmatter = false, trimLines = true, removeHorizontalRules = false } = opts;
|
|
16370
|
+
const changes = [];
|
|
16371
|
+
let s = md;
|
|
16372
|
+
if (stripFrontmatter) {
|
|
16373
|
+
const m = s.match(/^---\n[\s\S]*?\n---\n/);
|
|
16374
|
+
if (m) {
|
|
16375
|
+
s = s.slice(m[0].length);
|
|
16376
|
+
changes.push("Stripped frontmatter");
|
|
16377
|
+
}
|
|
16378
|
+
}
|
|
16379
|
+
if (removeComments) {
|
|
16380
|
+
const b = s.length;
|
|
16381
|
+
s = s.replace(/<!--[\s\S]*?-->/g, "");
|
|
16382
|
+
if (s.length < b) changes.push("Removed HTML comments");
|
|
16383
|
+
}
|
|
16384
|
+
if (trimLines) {
|
|
16385
|
+
const b = s;
|
|
16386
|
+
s = s.split("\n").map((l) => l.trimEnd()).join("\n");
|
|
16387
|
+
if (s !== b) changes.push("Trimmed trailing whitespace");
|
|
16388
|
+
}
|
|
16389
|
+
if (collapseBlankLines) {
|
|
16390
|
+
const b = s;
|
|
16391
|
+
s = s.replace(/\n{3,}/g, "\n\n");
|
|
16392
|
+
if (s !== b) changes.push("Collapsed blank lines");
|
|
16393
|
+
}
|
|
16394
|
+
if (removeHorizontalRules) {
|
|
16395
|
+
const b = s;
|
|
16396
|
+
s = s.replace(/^[-*_]{3,}\s*$/gm, "");
|
|
16397
|
+
if (s !== b) changes.push("Removed horizontal rules");
|
|
16398
|
+
}
|
|
16399
|
+
const lines = s.split("\n");
|
|
16400
|
+
const result = [];
|
|
16401
|
+
let inCode = false;
|
|
16402
|
+
for (const line of lines) {
|
|
16403
|
+
if (line.startsWith("```")) inCode = !inCode;
|
|
16404
|
+
if (!inCode && !line.startsWith(" ") && !line.startsWith(" ")) result.push(line.replace(/ {2,}/g, " "));
|
|
16405
|
+
else result.push(line);
|
|
16406
|
+
}
|
|
16407
|
+
s = result.join("\n").trim();
|
|
16408
|
+
const ot = Math.ceil(md.length / 4), ct = Math.ceil(s.length / 4);
|
|
16409
|
+
return { compressed: s, originalTokens: ot, compressedTokens: ct, savings: ot - ct, savingsPct: ot > 0 ? Math.round((ot - ct) / ot * 100) : 0, changes };
|
|
16410
|
+
}
|
|
16411
|
+
function _compressVal(val, opts, s) {
|
|
16412
|
+
if (val === null) return val;
|
|
16413
|
+
if (Array.isArray(val)) {
|
|
16414
|
+
return val.map((v) => _compressVal(v, opts, s)).filter((v) => {
|
|
16415
|
+
if (opts.removeNulls && v === null) {
|
|
16416
|
+
s.nulls++;
|
|
16417
|
+
return false;
|
|
16418
|
+
}
|
|
16419
|
+
if (opts.removeEmptyArrays && Array.isArray(v) && v.length === 0) {
|
|
16420
|
+
s.empArr++;
|
|
16421
|
+
return false;
|
|
16422
|
+
}
|
|
16423
|
+
if (opts.removeEmptyObjects && typeof v === "object" && v !== null && !Array.isArray(v) && Object.keys(v).length === 0) {
|
|
16424
|
+
s.empObj++;
|
|
16425
|
+
return false;
|
|
16426
|
+
}
|
|
16427
|
+
return true;
|
|
16428
|
+
});
|
|
16429
|
+
}
|
|
16430
|
+
if (typeof val === "object") {
|
|
16431
|
+
const obj = val;
|
|
16432
|
+
const keys = opts.sortKeys ? Object.keys(obj).sort() : Object.keys(obj);
|
|
16433
|
+
const r = {};
|
|
16434
|
+
for (const k of keys) {
|
|
16435
|
+
const v = _compressVal(obj[k], opts, s);
|
|
16436
|
+
if (opts.removeNulls && v === null) {
|
|
16437
|
+
s.nulls++;
|
|
16438
|
+
continue;
|
|
16439
|
+
}
|
|
16440
|
+
if (opts.removeEmptyArrays && Array.isArray(v) && v.length === 0) {
|
|
16441
|
+
s.empArr++;
|
|
16442
|
+
continue;
|
|
16443
|
+
}
|
|
16444
|
+
if (opts.removeEmptyObjects && typeof v === "object" && v !== null && !Array.isArray(v) && Object.keys(v).length === 0) {
|
|
16445
|
+
s.empObj++;
|
|
16446
|
+
continue;
|
|
16447
|
+
}
|
|
16448
|
+
r[k] = v;
|
|
16449
|
+
}
|
|
16450
|
+
return r;
|
|
16451
|
+
}
|
|
16452
|
+
return val;
|
|
16453
|
+
}
|
|
16454
|
+
function compressJson(json, opts = {}) {
|
|
16455
|
+
const { removeNulls = true, removeEmptyArrays = true, removeEmptyObjects = true, sortKeys = false } = opts;
|
|
16456
|
+
const origStr = JSON.stringify(json);
|
|
16457
|
+
const s = { nulls: 0, empArr: 0, empObj: 0 };
|
|
16458
|
+
const compressed = _compressVal(json, { removeNulls, removeEmptyArrays, removeEmptyObjects, sortKeys }, s);
|
|
16459
|
+
const compStr = JSON.stringify(compressed);
|
|
16460
|
+
const ot = Math.ceil(origStr.length / 4), ct = Math.ceil(compStr.length / 4);
|
|
16461
|
+
return { compressed: compStr, originalTokens: ot, compressedTokens: ct, savings: ot - ct, savingsPct: ot > 0 ? Math.round((ot - ct) / ot * 100) : 0, removedNulls: s.nulls, removedEmptyArrays: s.empArr, removedEmptyObjects: s.empObj };
|
|
16462
|
+
}
|
|
16463
|
+
var _RR_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "is", "are", "was", "were", "be", "been", "have", "has", "had", "it", "this", "that", "these", "those", "i", "we", "you", "he", "she", "they"]);
|
|
16464
|
+
function _tok(text) {
|
|
16465
|
+
return text.toLowerCase().match(/\b[a-z]{2,}\b/g)?.filter((w) => !_RR_STOP.has(w)) ?? [];
|
|
16466
|
+
}
|
|
16467
|
+
function rerankChunks(query, chunks, topK) {
|
|
16468
|
+
const qt = _tok(query), tc = chunks.map((c) => _tok(c));
|
|
16469
|
+
const idf = /* @__PURE__ */ new Map();
|
|
16470
|
+
for (const t of qt) {
|
|
16471
|
+
const df = tc.filter((c) => c.includes(t)).length;
|
|
16472
|
+
idf.set(t, Math.log((chunks.length + 1) / (df + 1)) + 1);
|
|
16473
|
+
}
|
|
16474
|
+
const ranked = chunks.map((text, index) => {
|
|
16475
|
+
const counts = /* @__PURE__ */ new Map();
|
|
16476
|
+
for (const t of tc[index]) counts.set(t, (counts.get(t) ?? 0) + 1);
|
|
16477
|
+
const max = Math.max(1, ...counts.values());
|
|
16478
|
+
let score = 0;
|
|
16479
|
+
const matched = [];
|
|
16480
|
+
for (const t of qt) {
|
|
16481
|
+
const s = (counts.get(t) ?? 0) / max * (idf.get(t) ?? 1);
|
|
16482
|
+
if (s > 0) {
|
|
16483
|
+
score += s;
|
|
16484
|
+
matched.push(t);
|
|
16485
|
+
}
|
|
16486
|
+
}
|
|
16487
|
+
return { index, text, score: Math.round(score * 1e3) / 1e3, matchedTerms: [...new Set(matched)] };
|
|
16488
|
+
});
|
|
16489
|
+
ranked.sort((a, b) => b.score - a.score);
|
|
16490
|
+
return { ranked: topK ? ranked.slice(0, topK) : ranked, query, totalChunks: chunks.length };
|
|
16491
|
+
}
|
|
16492
|
+
function scoreRelevance(query, passage) {
|
|
16493
|
+
const qt = [...new Set(_tok(query))], pt = _tok(passage);
|
|
16494
|
+
const counts = /* @__PURE__ */ new Map();
|
|
16495
|
+
for (const t of pt) counts.set(t, (counts.get(t) ?? 0) + 1);
|
|
16496
|
+
const max = Math.max(1, ...counts.values());
|
|
16497
|
+
let score = 0;
|
|
16498
|
+
const matched = [];
|
|
16499
|
+
for (const t of qt) {
|
|
16500
|
+
const c = counts.get(t) ?? 0;
|
|
16501
|
+
if (c > 0) {
|
|
16502
|
+
score += c / max * (1 / Math.log2(pt.length + 2));
|
|
16503
|
+
matched.push(t);
|
|
16504
|
+
}
|
|
16505
|
+
}
|
|
16506
|
+
const coverage = qt.length > 0 ? matched.length / qt.length : 0;
|
|
16507
|
+
const norm = Math.min(1, score * qt.length);
|
|
16508
|
+
const fs = Math.round((norm * 0.6 + coverage * 0.4) * 100) / 100;
|
|
16509
|
+
const grade = fs >= 0.6 ? "high" : fs >= 0.35 ? "medium" : fs > 0 ? "low" : "none";
|
|
16510
|
+
return { score: fs, grade, matchedTerms: matched, coverage: Math.round(coverage * 100) / 100 };
|
|
16511
|
+
}
|
|
16512
|
+
var _SYN = { fast: ["quick", "rapid"], slow: ["sluggish", "delayed"], error: ["bug", "fault", "exception"], fix: ["repair", "resolve", "patch"], search: ["query", "find", "retrieve"], document: ["file", "text", "content"], user: ["customer", "client"], data: ["information", "records"], api: ["endpoint", "service"], build: ["compile", "create"], delete: ["remove", "drop", "erase"], update: ["modify", "edit", "change"], create: ["add", "insert", "generate"], get: ["fetch", "retrieve", "obtain"], list: ["enumerate", "show", "index"], send: ["post", "submit", "transmit"], check: ["verify", "validate", "inspect"], large: ["big", "huge", "massive"], small: ["tiny", "minimal", "compact"], improve: ["enhance", "optimize", "boost"], analyze: ["examine", "evaluate", "assess"], model: ["llm", "ai", "classifier"], token: ["word", "subword", "unit"], chunk: ["segment", "fragment", "block"], embed: ["encode", "vectorize", "represent"], score: ["rank", "rate", "evaluate"], similar: ["related", "relevant", "matching"], agent: ["bot", "assistant", "worker"], memory: ["context", "history", "state"], cost: ["price", "expense", "fee"], deploy: ["release", "ship", "launch"] };
|
|
16513
|
+
var _QE_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "is", "are", "was"]);
|
|
16514
|
+
function expandQuery(query, maxSynonyms = 2) {
|
|
16515
|
+
const words = query.toLowerCase().match(/\b[a-z]{2,}\b/g) ?? [];
|
|
16516
|
+
const added = [];
|
|
16517
|
+
const terms = [...words];
|
|
16518
|
+
for (const w of words) {
|
|
16519
|
+
if (_QE_STOP.has(w)) continue;
|
|
16520
|
+
const s = _SYN[w];
|
|
16521
|
+
if (s) {
|
|
16522
|
+
for (const syn of s.slice(0, maxSynonyms)) if (!terms.includes(syn)) {
|
|
16523
|
+
terms.push(syn);
|
|
16524
|
+
added.push(syn);
|
|
16525
|
+
}
|
|
16526
|
+
}
|
|
16527
|
+
}
|
|
16528
|
+
for (const w of words) {
|
|
16529
|
+
let stem = "";
|
|
16530
|
+
if (w.endsWith("ing")) stem = w.slice(0, -3);
|
|
16531
|
+
else if (w.endsWith("ed")) stem = w.slice(0, -2);
|
|
16532
|
+
else if (w.endsWith("s") && !w.endsWith("ss")) stem = w.slice(0, -1);
|
|
16533
|
+
else if (w.endsWith("er")) stem = w.slice(0, -2);
|
|
16534
|
+
if (stem.length >= 3 && !terms.includes(stem)) {
|
|
16535
|
+
terms.push(stem);
|
|
16536
|
+
added.push(stem);
|
|
16537
|
+
}
|
|
16538
|
+
}
|
|
16539
|
+
return { original: query, expanded: terms.join(" "), terms, synonymsAdded: added };
|
|
16540
|
+
}
|
|
16541
|
+
var _SUM_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can", "not", "no", "it", "its", "this", "that", "these", "those", "i", "we", "you", "he", "she", "they", "their", "our", "your", "his", "her"]);
|
|
16542
|
+
function _sumTok(text) {
|
|
16543
|
+
return text.toLowerCase().match(/\b[a-z]{3,}\b/g)?.filter((w) => !_SUM_STOP.has(w)) ?? [];
|
|
16544
|
+
}
|
|
16545
|
+
function _splitSents(text) {
|
|
16546
|
+
return text.match(/[^.!?\n]+[.!?\n]+(\s|$)|[^.!?\n]+$/g)?.map((s) => s.trim()).filter(Boolean) ?? [text];
|
|
16547
|
+
}
|
|
16548
|
+
function summarizeForLlm(text, maxTokens = 200, strategy = "extractive") {
|
|
16549
|
+
const sents = _splitSents(text);
|
|
16550
|
+
const ot = Math.ceil(text.length / 4);
|
|
16551
|
+
const tok = (s) => Math.ceil(s.length / 4);
|
|
16552
|
+
if (strategy === "first") {
|
|
16553
|
+
let sum = "";
|
|
16554
|
+
for (const s of sents) {
|
|
16555
|
+
const c = sum ? sum + " " + s : s;
|
|
16556
|
+
if (tok(c) > maxTokens) break;
|
|
16557
|
+
sum = c;
|
|
16558
|
+
}
|
|
16559
|
+
const st2 = tok(sum);
|
|
16560
|
+
return { summary: sum, originalTokens: ot, summaryTokens: st2, compressionRatio: Math.round(st2 / Math.max(1, ot) * 100) / 100, sentencesKept: sum.split(/[.!?]/).filter(Boolean).length, sentencesTotal: sents.length };
|
|
16561
|
+
}
|
|
16562
|
+
if (strategy === "last") {
|
|
16563
|
+
const rev = [...sents].reverse();
|
|
16564
|
+
let sum = "";
|
|
16565
|
+
for (const s of rev) {
|
|
16566
|
+
const c = s + (sum ? " " + sum : "");
|
|
16567
|
+
if (tok(c) > maxTokens) break;
|
|
16568
|
+
sum = c;
|
|
16569
|
+
}
|
|
16570
|
+
const st2 = tok(sum);
|
|
16571
|
+
return { summary: sum, originalTokens: ot, summaryTokens: st2, compressionRatio: Math.round(st2 / Math.max(1, ot) * 100) / 100, sentencesKept: sum.split(/[.!?]/).filter(Boolean).length, sentencesTotal: sents.length };
|
|
16572
|
+
}
|
|
16573
|
+
const allTok = _sumTok(text);
|
|
16574
|
+
const freq = /* @__PURE__ */ new Map();
|
|
16575
|
+
for (const t of allTok) freq.set(t, (freq.get(t) ?? 0) + 1);
|
|
16576
|
+
const maxF = Math.max(1, ...freq.values());
|
|
16577
|
+
const scored = sents.map((s, i) => {
|
|
16578
|
+
const ts = _sumTok(s);
|
|
16579
|
+
const sc = ts.reduce((sum, t) => sum + (freq.get(t) ?? 0) / maxF, 0) / Math.max(1, ts.length);
|
|
16580
|
+
return { s, i, score: sc + (i === 0 || i === sents.length - 1 ? 0.1 : 0) };
|
|
16581
|
+
});
|
|
16582
|
+
const picked = [];
|
|
16583
|
+
let used = 0;
|
|
16584
|
+
for (const { s, i } of [...scored].sort((a, b) => b.score - a.score)) {
|
|
16585
|
+
const t = tok(s);
|
|
16586
|
+
if (used + t > maxTokens) continue;
|
|
16587
|
+
picked.push({ s, i });
|
|
16588
|
+
used += t;
|
|
16589
|
+
}
|
|
16590
|
+
picked.sort((a, b) => a.i - b.i);
|
|
16591
|
+
const summary = picked.map((p) => p.s).join(" ");
|
|
16592
|
+
const st = tok(summary);
|
|
16593
|
+
return { summary, originalTokens: ot, summaryTokens: st, compressionRatio: Math.round(st / Math.max(1, ot) * 100) / 100, sentencesKept: picked.length, sentencesTotal: sents.length };
|
|
16594
|
+
}
|
|
15448
16595
|
|
|
15449
16596
|
// src/index.ts
|
|
15450
16597
|
var version = "0.1.0";
|
|
15451
16598
|
|
|
16599
|
+
exports.ai_agent = ai_agent_exports;
|
|
15452
16600
|
exports.api = api_exports;
|
|
15453
16601
|
exports.code = code_exports;
|
|
15454
16602
|
exports.color = color_exports;
|