@mrciphersmith/keryx 0.3.76 → 0.3.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1304 -123
- package/dist/core.js +1014 -59
- package/package.json +3 -3
package/dist/cli.js
CHANGED
|
@@ -1757,13 +1757,366 @@ function isValidIp(value) {
|
|
|
1757
1757
|
}
|
|
1758
1758
|
return false;
|
|
1759
1759
|
}
|
|
1760
|
-
function
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1760
|
+
function isWordBoundary(flavour, content, index) {
|
|
1761
|
+
return flavour.wordChar.test(wordCharBefore(content, index)) !== flavour.wordChar.test(wordCharAt(content, index));
|
|
1762
|
+
}
|
|
1763
|
+
function wordCharBefore(content, index) {
|
|
1764
|
+
if (index <= 0) {
|
|
1765
|
+
return "";
|
|
1765
1766
|
}
|
|
1766
|
-
|
|
1767
|
+
const low = content.charCodeAt(index - 1);
|
|
1768
|
+
if (low >= 56320 && low <= 57343 && index >= 2) {
|
|
1769
|
+
const high = content.charCodeAt(index - 2);
|
|
1770
|
+
if (high >= 55296 && high <= 56319) {
|
|
1771
|
+
return content.slice(index - 2, index);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
return content[index - 1];
|
|
1775
|
+
}
|
|
1776
|
+
function wordCharAt(content, index) {
|
|
1777
|
+
const code = content.codePointAt(index);
|
|
1778
|
+
return code === undefined ? "" : String.fromCodePoint(code);
|
|
1779
|
+
}
|
|
1780
|
+
function execEmail(flavour, content, from) {
|
|
1781
|
+
let at = content.indexOf("@", from);
|
|
1782
|
+
while (at !== -1) {
|
|
1783
|
+
let start = at;
|
|
1784
|
+
while (start > from && flavour.localChar.test(wordCharBefore(content, start))) {
|
|
1785
|
+
start -= wordCharBefore(content, start).length;
|
|
1786
|
+
}
|
|
1787
|
+
while (start < at && !isWordBoundary(flavour, content, start)) {
|
|
1788
|
+
start += wordCharAt(content, start).length;
|
|
1789
|
+
}
|
|
1790
|
+
if (start < at) {
|
|
1791
|
+
flavour.from.lastIndex = start;
|
|
1792
|
+
const m = flavour.from.exec(content);
|
|
1793
|
+
if (m !== null) {
|
|
1794
|
+
return m;
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
at = content.indexOf("@", at + 1);
|
|
1798
|
+
}
|
|
1799
|
+
return null;
|
|
1800
|
+
}
|
|
1801
|
+
function emailScannerFor(flavour) {
|
|
1802
|
+
return {
|
|
1803
|
+
lastIndex: 0,
|
|
1804
|
+
exec(content) {
|
|
1805
|
+
const m = execEmail(flavour, content, this.lastIndex);
|
|
1806
|
+
this.lastIndex = m === null ? 0 : m.index + m[0].length;
|
|
1807
|
+
return m;
|
|
1808
|
+
}
|
|
1809
|
+
};
|
|
1810
|
+
}
|
|
1811
|
+
function execPhone(content, from) {
|
|
1812
|
+
let pos = from;
|
|
1813
|
+
while (pos < content.length) {
|
|
1814
|
+
const digitAt = content[pos] === "+" ? pos + 1 : pos;
|
|
1815
|
+
if (!DIGIT.test(content[digitAt] ?? "") || pos > 0 && PHONE_START_BLOCKER.test(content[pos - 1])) {
|
|
1816
|
+
pos += 1;
|
|
1817
|
+
continue;
|
|
1818
|
+
}
|
|
1819
|
+
PHONE_FROM.lastIndex = pos;
|
|
1820
|
+
const m = PHONE_FROM.exec(content);
|
|
1821
|
+
if (m !== null) {
|
|
1822
|
+
return m;
|
|
1823
|
+
}
|
|
1824
|
+
pos = digitAt + 1;
|
|
1825
|
+
while (pos < content.length && PHONE_RUN_CHAR.test(content[pos])) {
|
|
1826
|
+
pos += 1;
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
return null;
|
|
1830
|
+
}
|
|
1831
|
+
function phoneVerdict(content, start, end) {
|
|
1832
|
+
let digits = 0;
|
|
1833
|
+
for (let i = start;i < end && digits <= PHONE_MAX_DIGITS; i += 1) {
|
|
1834
|
+
if (DIGIT.test(content[i])) {
|
|
1835
|
+
digits += 1;
|
|
1836
|
+
}
|
|
1837
|
+
}
|
|
1838
|
+
if (digits < PHONE_MIN_DIGITS || digits > PHONE_MAX_DIGITS) {
|
|
1839
|
+
return false;
|
|
1840
|
+
}
|
|
1841
|
+
const value = content.slice(start, end);
|
|
1842
|
+
return hasPhoneSeparatorShape(value) && !containsCalendarDate(value) && !isIdentifierFragment(content, start, end);
|
|
1843
|
+
}
|
|
1844
|
+
function hasPhoneEvidence(value, singleToken) {
|
|
1845
|
+
return PHONE_SEPARATOR.test(value) || singleToken && value.includes(".") || PHONE_NANP_SPACED.test(value);
|
|
1846
|
+
}
|
|
1847
|
+
function phoneTokens(content, from, to) {
|
|
1848
|
+
const tokens = [];
|
|
1849
|
+
let i = from;
|
|
1850
|
+
while (i < to) {
|
|
1851
|
+
const gapStart = i;
|
|
1852
|
+
while (i < to && WHITESPACE.test(content[i])) {
|
|
1853
|
+
i += 1;
|
|
1854
|
+
}
|
|
1855
|
+
if (i >= to) {
|
|
1856
|
+
break;
|
|
1857
|
+
}
|
|
1858
|
+
const breakBefore = tokens.length > 0 && (i - gapStart >= 2 || LINE_BREAK.test(content.slice(gapStart, i)));
|
|
1859
|
+
const start = i;
|
|
1860
|
+
let digits = 0;
|
|
1861
|
+
let lastDigit = -1;
|
|
1862
|
+
let coreStart = -1;
|
|
1863
|
+
let leadDigits = 0;
|
|
1864
|
+
while (i < to && !WHITESPACE.test(content[i])) {
|
|
1865
|
+
const isDigit = DIGIT.test(content[i]);
|
|
1866
|
+
if (coreStart < 0) {
|
|
1867
|
+
const startsHere = (isDigit || content[i] === "+" && i + 1 < to && DIGIT.test(content[i + 1])) && (i === 0 || !PHONE_START_BLOCKER.test(content[i - 1]));
|
|
1868
|
+
if (startsHere) {
|
|
1869
|
+
coreStart = i;
|
|
1870
|
+
} else if (isDigit) {
|
|
1871
|
+
leadDigits += 1;
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
if (isDigit) {
|
|
1875
|
+
digits += 1;
|
|
1876
|
+
lastDigit = i;
|
|
1877
|
+
}
|
|
1878
|
+
i += 1;
|
|
1879
|
+
}
|
|
1880
|
+
const text = content.slice(start, i);
|
|
1881
|
+
const groupable = (PHONE_DIAL_GROUP.test(text) || PHONE_FIRST_GROUP.test(text)) && !containsCalendarDate(text);
|
|
1882
|
+
tokens.push({ start, end: i, coreStart, coreEnd: lastDigit + 1, leadDigits, digits, breakBefore, groupable });
|
|
1883
|
+
}
|
|
1884
|
+
return tokens;
|
|
1885
|
+
}
|
|
1886
|
+
function rescanPhoneRun(content, from, to) {
|
|
1887
|
+
const tokens = phoneTokens(content, from, to);
|
|
1888
|
+
const found = [];
|
|
1889
|
+
const chainAfter = new Array(tokens.length).fill(0);
|
|
1890
|
+
for (let i = tokens.length - 2;i >= 0; i -= 1) {
|
|
1891
|
+
const next = tokens[i + 1];
|
|
1892
|
+
chainAfter[i] = next.groupable && !next.breakBefore ? chainAfter[i + 1] + next.digits : 0;
|
|
1893
|
+
}
|
|
1894
|
+
let first = 0;
|
|
1895
|
+
let resume = -1;
|
|
1896
|
+
while (first < tokens.length) {
|
|
1897
|
+
const head = tokens[first];
|
|
1898
|
+
if (head.coreStart < 0 || head.coreEnd <= head.coreStart) {
|
|
1899
|
+
first += 1;
|
|
1900
|
+
continue;
|
|
1901
|
+
}
|
|
1902
|
+
const startsRun = first === 0 || first === resume || head.breakBefore || !tokens[first - 1].groupable;
|
|
1903
|
+
let longest = -1;
|
|
1904
|
+
let clean = -1;
|
|
1905
|
+
const consider = (last) => {
|
|
1906
|
+
const end = tokens[last].coreEnd;
|
|
1907
|
+
if (hasPhoneEvidence(content.slice(head.coreStart, end), last === first) && phoneVerdict(content, head.coreStart, end)) {
|
|
1908
|
+
longest = last;
|
|
1909
|
+
const rest = chainAfter[last];
|
|
1910
|
+
if (rest === 0 || rest >= PHONE_MIN_DIGITS) {
|
|
1911
|
+
clean = last;
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
};
|
|
1915
|
+
let digits = head.digits - head.leadDigits;
|
|
1916
|
+
if (digits >= PHONE_MIN_DIGITS) {
|
|
1917
|
+
consider(first);
|
|
1918
|
+
}
|
|
1919
|
+
const headGroup = content.slice(head.coreStart, head.end);
|
|
1920
|
+
let extendable = startsRun && (PHONE_FIRST_GROUP.test(headGroup) || PHONE_DIAL_GROUP.test(headGroup));
|
|
1921
|
+
for (let last = first + 1;last < tokens.length && extendable; last += 1) {
|
|
1922
|
+
const tail = tokens[last];
|
|
1923
|
+
digits += tail.digits;
|
|
1924
|
+
if (digits > PHONE_MAX_DIGITS || tail.digits === 0 || tail.breakBefore) {
|
|
1925
|
+
break;
|
|
1926
|
+
}
|
|
1927
|
+
if (digits >= PHONE_MIN_DIGITS && PHONE_DIAL_GROUP.test(content.slice(tail.start, tail.coreEnd))) {
|
|
1928
|
+
consider(last);
|
|
1929
|
+
}
|
|
1930
|
+
extendable = PHONE_DIAL_GROUP.test(content.slice(tail.start, tail.end));
|
|
1931
|
+
}
|
|
1932
|
+
const best = clean >= 0 ? clean : longest;
|
|
1933
|
+
if (best >= 0) {
|
|
1934
|
+
const value = content.slice(head.coreStart, tokens[best].coreEnd);
|
|
1935
|
+
found.push({ index: head.coreStart, 0: value, 1: value });
|
|
1936
|
+
first = best + 1;
|
|
1937
|
+
resume = first;
|
|
1938
|
+
} else {
|
|
1939
|
+
first += 1;
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
return found;
|
|
1943
|
+
}
|
|
1944
|
+
function joinedPhones(content, from, to) {
|
|
1945
|
+
const found = [];
|
|
1946
|
+
let i = from;
|
|
1947
|
+
while (i < to) {
|
|
1948
|
+
while (i < to && WHITESPACE.test(content[i])) {
|
|
1949
|
+
i += 1;
|
|
1950
|
+
}
|
|
1951
|
+
let zoneEnd = i;
|
|
1952
|
+
let total = 0;
|
|
1953
|
+
while (zoneEnd < to && !WHITESPACE.test(content[zoneEnd])) {
|
|
1954
|
+
if (DIGIT.test(content[zoneEnd])) {
|
|
1955
|
+
total += 1;
|
|
1956
|
+
}
|
|
1957
|
+
zoneEnd += 1;
|
|
1958
|
+
}
|
|
1959
|
+
if (total > PHONE_MAX_DIGITS && !containsCalendarDate(content.slice(i, zoneEnd))) {
|
|
1960
|
+
joinedZone(content, i, zoneEnd, total, found);
|
|
1961
|
+
}
|
|
1962
|
+
i = zoneEnd;
|
|
1963
|
+
}
|
|
1964
|
+
return found;
|
|
1965
|
+
}
|
|
1966
|
+
function isJoinerAt(content, k, to) {
|
|
1967
|
+
const ch = content[k];
|
|
1968
|
+
return (ch === "-" || ch === ".") && k > 0 && DIGIT.test(content[k - 1]) && k + 1 < to && DIGIT.test(content[k + 1]);
|
|
1969
|
+
}
|
|
1970
|
+
function joinedZone(content, zoneStart, zoneEnd, total, found) {
|
|
1971
|
+
let s = zoneStart;
|
|
1972
|
+
let before = 0;
|
|
1973
|
+
for (;; ) {
|
|
1974
|
+
while (s < zoneEnd && !DIGIT.test(content[s]) && !(content[s] === "+" && s + 1 < zoneEnd && DIGIT.test(content[s + 1]))) {
|
|
1975
|
+
s += 1;
|
|
1976
|
+
}
|
|
1977
|
+
if (s >= zoneEnd) {
|
|
1978
|
+
return;
|
|
1979
|
+
}
|
|
1980
|
+
let digits = 0;
|
|
1981
|
+
let lastDigit = s;
|
|
1982
|
+
let open = true;
|
|
1983
|
+
const cuts = [];
|
|
1984
|
+
for (let k = s;k < zoneEnd && open; k += 1) {
|
|
1985
|
+
if (DIGIT.test(content[k])) {
|
|
1986
|
+
digits += 1;
|
|
1987
|
+
lastDigit = k;
|
|
1988
|
+
open = digits <= PHONE_MAX_DIGITS;
|
|
1989
|
+
} else if (digits >= PHONE_MIN_DIGITS && isJoinerAt(content, k, zoneEnd)) {
|
|
1990
|
+
cuts.push({ end: k, digits });
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
if (open && digits >= PHONE_MIN_DIGITS) {
|
|
1994
|
+
cuts.push({ end: lastDigit + 1, digits });
|
|
1995
|
+
}
|
|
1996
|
+
let pick = -1;
|
|
1997
|
+
for (let c = cuts.length - 1;c >= 0; c -= 1) {
|
|
1998
|
+
const cut = cuts[c];
|
|
1999
|
+
const piece = content.slice(s, cut.end);
|
|
2000
|
+
if (!PHONE_JOINED_PIECE.test(piece) || PHONE_CARD_SHAPE.test(piece) || !phoneVerdict(content, s, cut.end)) {
|
|
2001
|
+
continue;
|
|
2002
|
+
}
|
|
2003
|
+
if (pick < 0) {
|
|
2004
|
+
pick = c;
|
|
2005
|
+
}
|
|
2006
|
+
const rest = total - before - cut.digits;
|
|
2007
|
+
if (rest === 0 || rest >= PHONE_MIN_DIGITS) {
|
|
2008
|
+
pick = c;
|
|
2009
|
+
break;
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
if (pick < 0) {
|
|
2013
|
+
return;
|
|
2014
|
+
}
|
|
2015
|
+
const cut = cuts[pick];
|
|
2016
|
+
const value = content.slice(s, cut.end);
|
|
2017
|
+
found.push({ index: s, 0: value, 1: value });
|
|
2018
|
+
before += cut.digits;
|
|
2019
|
+
s = cut.end + 1;
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
function withJoined(joined, rescans) {
|
|
2023
|
+
if (joined.length === 0) {
|
|
2024
|
+
return rescans;
|
|
2025
|
+
}
|
|
2026
|
+
const merged = [];
|
|
2027
|
+
let cursor = 0;
|
|
2028
|
+
let pending = 0;
|
|
2029
|
+
for (const rescan of rescans) {
|
|
2030
|
+
const rescanEnd = rescan.index + (rescan[0] ?? "").length;
|
|
2031
|
+
while (cursor < joined.length && joined[cursor].index + (joined[cursor][0] ?? "").length <= rescan.index) {
|
|
2032
|
+
cursor += 1;
|
|
2033
|
+
}
|
|
2034
|
+
if (cursor < joined.length && joined[cursor].index < rescanEnd) {
|
|
2035
|
+
continue;
|
|
2036
|
+
}
|
|
2037
|
+
while (pending < joined.length && joined[pending].index < rescan.index) {
|
|
2038
|
+
merged.push(joined[pending]);
|
|
2039
|
+
pending += 1;
|
|
2040
|
+
}
|
|
2041
|
+
merged.push(rescan);
|
|
2042
|
+
}
|
|
2043
|
+
while (pending < joined.length) {
|
|
2044
|
+
merged.push(joined[pending]);
|
|
2045
|
+
pending += 1;
|
|
2046
|
+
}
|
|
2047
|
+
return merged;
|
|
2048
|
+
}
|
|
2049
|
+
function phoneBaselineEnd(content, runEnd) {
|
|
2050
|
+
for (let e = runEnd;e > 0; e -= 1) {
|
|
2051
|
+
if (DIGIT.test(content[e - 1]) && (e >= content.length || !PHONE_START_BLOCKER.test(content[e]))) {
|
|
2052
|
+
return e;
|
|
2053
|
+
}
|
|
2054
|
+
}
|
|
2055
|
+
return -1;
|
|
2056
|
+
}
|
|
2057
|
+
function isIbanChar(code) {
|
|
2058
|
+
return code >= 65 && code <= 90 || code >= 48 && code <= 57;
|
|
2059
|
+
}
|
|
2060
|
+
function isAsciiWordChar(code) {
|
|
2061
|
+
return isIbanChar(code) || code >= 97 && code <= 122 || code === 95;
|
|
2062
|
+
}
|
|
2063
|
+
function ibanEnd(content, base, pos, groups) {
|
|
2064
|
+
const key = (pos - base) * (IBAN_MAX_GROUPS + 1) + groups;
|
|
2065
|
+
if (ibanFailed[key] === ibanStamp) {
|
|
2066
|
+
return -1;
|
|
2067
|
+
}
|
|
2068
|
+
if (groups < IBAN_MAX_GROUPS) {
|
|
2069
|
+
const from = content.charCodeAt(pos) === 32 ? pos + 1 : pos;
|
|
2070
|
+
let run = 0;
|
|
2071
|
+
while (run < 4 && isIbanChar(content.charCodeAt(from + run))) {
|
|
2072
|
+
run += 1;
|
|
2073
|
+
}
|
|
2074
|
+
for (let length = run;length >= 1; length -= 1) {
|
|
2075
|
+
const end = ibanEnd(content, base, from + length, groups + 1);
|
|
2076
|
+
if (end >= 0) {
|
|
2077
|
+
return end;
|
|
2078
|
+
}
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
if (groups >= 2 && !isAsciiWordChar(content.charCodeAt(pos))) {
|
|
2082
|
+
return pos;
|
|
2083
|
+
}
|
|
2084
|
+
ibanFailed[key] = ibanStamp;
|
|
2085
|
+
return -1;
|
|
2086
|
+
}
|
|
2087
|
+
function isCardCandidate(value) {
|
|
2088
|
+
const separators = value.match(CARD_GROUP_SEPARATOR) ?? [];
|
|
2089
|
+
const first = separators[0];
|
|
2090
|
+
if (first === undefined) {
|
|
2091
|
+
return true;
|
|
2092
|
+
}
|
|
2093
|
+
if (!separators.every((separator) => separator === first)) {
|
|
2094
|
+
return false;
|
|
2095
|
+
}
|
|
2096
|
+
const lengths = value.split(first).map((group) => group.length);
|
|
2097
|
+
const last = lengths[lengths.length - 1];
|
|
2098
|
+
if (lengths.length === 3 && lengths[0] === 4 && lengths[1] === 6 && (last === 5 || last === 4)) {
|
|
2099
|
+
return true;
|
|
2100
|
+
}
|
|
2101
|
+
return lengths.slice(0, -1).every((length) => length === 4) && last >= 1 && last <= 7;
|
|
2102
|
+
}
|
|
2103
|
+
function hasNonHexGlue(content, start, end) {
|
|
2104
|
+
const before = GLUE_LETTER_BEFORE.exec(content.slice(Math.max(0, start - GLUE_FLANK), start))?.[0] ?? "";
|
|
2105
|
+
const after = GLUE_LETTER_AFTER.exec(content.slice(end, end + GLUE_FLANK))?.[0] ?? "";
|
|
2106
|
+
return before === "" && after === "" || NON_HEX_LETTER.test(before) || NON_HEX_LETTER.test(after);
|
|
2107
|
+
}
|
|
2108
|
+
function outsideUuid(content, start, end) {
|
|
2109
|
+
const from = Math.max(0, start - UUID_REACH);
|
|
2110
|
+
const window = content.slice(from, Math.min(content.length, end + UUID_REACH));
|
|
2111
|
+
UUID_ANYWHERE.lastIndex = 0;
|
|
2112
|
+
let m;
|
|
2113
|
+
while ((m = UUID_ANYWHERE.exec(window)) !== null) {
|
|
2114
|
+
if (from + m.index < end && from + m.index + m[0].length > start) {
|
|
2115
|
+
return false;
|
|
2116
|
+
}
|
|
2117
|
+
UUID_ANYWHERE.lastIndex = m.index + 1;
|
|
2118
|
+
}
|
|
2119
|
+
return true;
|
|
1767
2120
|
}
|
|
1768
2121
|
function containsCalendarDate(value) {
|
|
1769
2122
|
return CALENDAR_DATE.test(value);
|
|
@@ -1816,71 +2169,644 @@ function hasPhoneSeparatorShape(value) {
|
|
|
1816
2169
|
return /^\(?\d{2,4}\)?(?:[-.]\d{2,4})*$/.test(group);
|
|
1817
2170
|
});
|
|
1818
2171
|
}
|
|
1819
|
-
function
|
|
2172
|
+
function scanWith(content, rule, scanner, validate, accept) {
|
|
1820
2173
|
const matches = [];
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
2174
|
+
scanner.reset?.();
|
|
2175
|
+
scanner.lastIndex = 0;
|
|
2176
|
+
let m;
|
|
2177
|
+
while ((m = scanner.exec(content)) !== null) {
|
|
2178
|
+
const group = rule.valueGroup ?? 0;
|
|
2179
|
+
const value = m[group];
|
|
2180
|
+
if (value === undefined || value.length === 0) {
|
|
2181
|
+
continue;
|
|
2182
|
+
}
|
|
2183
|
+
if (validate && !validate(value)) {
|
|
2184
|
+
continue;
|
|
2185
|
+
}
|
|
2186
|
+
const start = group === 0 ? m.index : m.index + m[0].indexOf(value);
|
|
2187
|
+
const end = start + value.length;
|
|
2188
|
+
if (rule.accept && !rule.accept(content, start, end) || accept && !accept(content, start, end)) {
|
|
2189
|
+
if (m.index === scanner.lastIndex) {
|
|
2190
|
+
scanner.lastIndex += 1;
|
|
2191
|
+
}
|
|
2192
|
+
continue;
|
|
2193
|
+
}
|
|
2194
|
+
const match = {
|
|
2195
|
+
category: "pii",
|
|
2196
|
+
policyId: rule.policyId,
|
|
2197
|
+
severity: rule.severity,
|
|
2198
|
+
confidence: rule.confidence,
|
|
2199
|
+
start,
|
|
2200
|
+
end,
|
|
2201
|
+
value,
|
|
2202
|
+
mask: rule.mask,
|
|
2203
|
+
remediation: "Redact personal data before persisting or publishing."
|
|
2204
|
+
};
|
|
2205
|
+
if (rescanned.has(m)) {
|
|
2206
|
+
rescanTag.add(match);
|
|
2207
|
+
}
|
|
2208
|
+
matches.push(match);
|
|
2209
|
+
if (m.index === scanner.lastIndex) {
|
|
2210
|
+
scanner.lastIndex += 1;
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
return matches;
|
|
2214
|
+
}
|
|
2215
|
+
function scanRule(content, rule) {
|
|
2216
|
+
const matches = scanWith(content, rule, rule.regex, rule.validate, undefined);
|
|
2217
|
+
if (rule.also === undefined) {
|
|
2218
|
+
return matches;
|
|
2219
|
+
}
|
|
2220
|
+
let result = matches;
|
|
2221
|
+
for (const extra of rule.also) {
|
|
2222
|
+
const found = scanWith(content, rule, extra.regex, extra.validate ?? rule.validate, extra.accept);
|
|
2223
|
+
const added = found.filter((match) => !containedIn(result, match.start, match.end));
|
|
2224
|
+
if (added.length > 0) {
|
|
2225
|
+
result = coalesce(content, [...result, ...added].sort((x, y) => x.start - y.start || y.end - x.end));
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
return result;
|
|
2229
|
+
}
|
|
2230
|
+
function scanRules(content) {
|
|
2231
|
+
return RULES.map((rule) => scanRule(content, rule));
|
|
2232
|
+
}
|
|
2233
|
+
function decimalDigitValue(codePoint) {
|
|
2234
|
+
if (codePoint < 1632) {
|
|
2235
|
+
return -1;
|
|
2236
|
+
}
|
|
2237
|
+
const cached = digitValues.get(codePoint);
|
|
2238
|
+
if (cached !== undefined) {
|
|
2239
|
+
return cached;
|
|
2240
|
+
}
|
|
2241
|
+
let value = -1;
|
|
2242
|
+
if (DECIMAL_DIGIT.test(String.fromCodePoint(codePoint))) {
|
|
2243
|
+
let runStart = codePoint;
|
|
2244
|
+
while (runStart > 0 && DECIMAL_DIGIT.test(String.fromCodePoint(runStart - 1))) {
|
|
2245
|
+
runStart -= 1;
|
|
2246
|
+
}
|
|
2247
|
+
value = (codePoint - runStart) % 10;
|
|
2248
|
+
}
|
|
2249
|
+
digitValues.set(codePoint, value);
|
|
2250
|
+
return value;
|
|
2251
|
+
}
|
|
2252
|
+
function isHyphenLookalike(code) {
|
|
2253
|
+
return code >= 8208 && code <= 8213 || code === 8722 || code === 8315 || code === 8331 || code === 65112 || code === 65123 || code === 65293 || code === 1418 || code === 1470 || code === 6150 || code === 8259 || code === 12540 || code === 12448 || code === 11834 || code === 11835;
|
|
2254
|
+
}
|
|
2255
|
+
function isInvisible(code) {
|
|
2256
|
+
return code === 173 || code >= 8203 && code <= 8205 || code >= 8288 && code <= 8292 || code === 6158 || code === 847 || code === 8206 || code === 8207;
|
|
2257
|
+
}
|
|
2258
|
+
function foldNumberChar(code, options) {
|
|
2259
|
+
if (isInvisible(code)) {
|
|
2260
|
+
return code === 173 && options.shyHyphen ? "-" : "";
|
|
2261
|
+
}
|
|
2262
|
+
if (code === 160 || code === 5760 || code >= 8192 && code <= 8202 || code === 8232 || code === 8233 || code === 8239 || code === 8287 || code === 12288 || code === 65279) {
|
|
2263
|
+
return " ";
|
|
2264
|
+
}
|
|
2265
|
+
if (isHyphenLookalike(code)) {
|
|
2266
|
+
return "-";
|
|
2267
|
+
}
|
|
2268
|
+
if (code === 8228 || code === 65106 || code === 65294 || code === 12290 || code === 65377) {
|
|
2269
|
+
return ".";
|
|
2270
|
+
}
|
|
2271
|
+
if (code === 65312 || code === 65131) {
|
|
2272
|
+
return "@";
|
|
2273
|
+
}
|
|
2274
|
+
if (code === 65291 || code === 8314 || code === 8330 || code === 65122) {
|
|
2275
|
+
return "+";
|
|
2276
|
+
}
|
|
2277
|
+
if (code === 65288 || code === 8317 || code === 8333 || code === 65113) {
|
|
2278
|
+
return "(";
|
|
2279
|
+
}
|
|
2280
|
+
if (code === 65289 || code === 8318 || code === 8334 || code === 65114) {
|
|
2281
|
+
return ")";
|
|
2282
|
+
}
|
|
2283
|
+
if (options.letters && code >= 65313 && code <= 65338) {
|
|
2284
|
+
return String.fromCharCode(code - 65313 + 65);
|
|
2285
|
+
}
|
|
2286
|
+
const digit = decimalDigitValue(code);
|
|
2287
|
+
return digit < 0 ? null : String(digit);
|
|
2288
|
+
}
|
|
2289
|
+
function isNumberBreak(content, from, to) {
|
|
2290
|
+
const before = content[from - 1];
|
|
2291
|
+
const after = content[to];
|
|
2292
|
+
return from > 1 && (before === "-" || before === ".") && DIGIT.test(content[from - 2]) && DIGIT.test(after ?? "") || from > 0 && DIGIT.test(before ?? "") && (after === "-" || after === ".") && DIGIT.test(content[to + 1] ?? "");
|
|
2293
|
+
}
|
|
2294
|
+
function normalizeWindow(content, from, to, options) {
|
|
2295
|
+
const origin = new Int32Array(to - from + 1);
|
|
2296
|
+
const parts = [];
|
|
2297
|
+
let length = 0;
|
|
2298
|
+
let flushed = from;
|
|
2299
|
+
let changed = false;
|
|
2300
|
+
for (let i = from;i < to; ) {
|
|
2301
|
+
const code = content.charCodeAt(i);
|
|
2302
|
+
if (code < 128) {
|
|
2303
|
+
if (options.breaks && (code === 9 || code === 10 || code === 13)) {
|
|
2304
|
+
let j = i + 1;
|
|
2305
|
+
while (j < to && BREAK_CHAR.test(content[j])) {
|
|
2306
|
+
j += 1;
|
|
2307
|
+
}
|
|
2308
|
+
if (isNumberBreak(content, i, j)) {
|
|
2309
|
+
changed = true;
|
|
2310
|
+
parts.push(content.slice(flushed, i));
|
|
2311
|
+
flushed = j;
|
|
2312
|
+
} else {
|
|
2313
|
+
for (let k = i;k < j; k += 1) {
|
|
2314
|
+
origin[length++] = k;
|
|
2315
|
+
}
|
|
2316
|
+
}
|
|
2317
|
+
i = j;
|
|
1828
2318
|
continue;
|
|
1829
2319
|
}
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
2320
|
+
origin[length++] = i;
|
|
2321
|
+
i += 1;
|
|
2322
|
+
continue;
|
|
2323
|
+
}
|
|
2324
|
+
const codePoint = content.codePointAt(i);
|
|
2325
|
+
const width = codePoint > 65535 ? 2 : 1;
|
|
2326
|
+
const folded = foldNumberChar(codePoint, options);
|
|
2327
|
+
if (folded === null) {
|
|
2328
|
+
for (let k = 0;k < width; k += 1) {
|
|
2329
|
+
origin[length++] = i;
|
|
2330
|
+
}
|
|
2331
|
+
} else {
|
|
2332
|
+
changed = true;
|
|
2333
|
+
parts.push(content.slice(flushed, i));
|
|
2334
|
+
flushed = i + width;
|
|
2335
|
+
if (folded !== "") {
|
|
2336
|
+
parts.push(folded);
|
|
2337
|
+
origin[length++] = i;
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
i += width;
|
|
2341
|
+
}
|
|
2342
|
+
if (!changed) {
|
|
2343
|
+
return null;
|
|
2344
|
+
}
|
|
2345
|
+
parts.push(content.slice(flushed, to));
|
|
2346
|
+
origin[length] = to;
|
|
2347
|
+
return { text: parts.join(""), origin: origin.subarray(0, length + 1) };
|
|
2348
|
+
}
|
|
2349
|
+
function toOriginal(window, content, start, end) {
|
|
2350
|
+
const last = window.origin[end - 1];
|
|
2351
|
+
const width = content.codePointAt(last) > 65535 ? 2 : 1;
|
|
2352
|
+
return [window.origin[start], last + width];
|
|
2353
|
+
}
|
|
2354
|
+
function containedIn(spans, start, end) {
|
|
2355
|
+
let low = 0;
|
|
2356
|
+
let high = spans.length - 1;
|
|
2357
|
+
while (low <= high) {
|
|
2358
|
+
const mid = low + high >> 1;
|
|
2359
|
+
if (spans[mid].start <= start) {
|
|
2360
|
+
low = mid + 1;
|
|
2361
|
+
} else {
|
|
2362
|
+
high = mid - 1;
|
|
2363
|
+
}
|
|
2364
|
+
}
|
|
2365
|
+
const candidate = spans[high];
|
|
2366
|
+
return candidate !== undefined && candidate.start <= start && candidate.end >= end;
|
|
2367
|
+
}
|
|
2368
|
+
function coalesce(content, spans) {
|
|
2369
|
+
const merged = [];
|
|
2370
|
+
for (const span of spans) {
|
|
2371
|
+
const last = merged[merged.length - 1];
|
|
2372
|
+
if (last === undefined || span.start >= last.end) {
|
|
2373
|
+
merged.push(span);
|
|
2374
|
+
continue;
|
|
2375
|
+
}
|
|
2376
|
+
const end = Math.max(last.end, span.end);
|
|
2377
|
+
const union = { ...last, end, value: content.slice(last.start, end) };
|
|
2378
|
+
if (rescanTag.has(last) && rescanTag.has(span)) {
|
|
2379
|
+
rescanTag.add(union);
|
|
2380
|
+
}
|
|
2381
|
+
merged[merged.length - 1] = union;
|
|
2382
|
+
}
|
|
2383
|
+
return merged;
|
|
2384
|
+
}
|
|
2385
|
+
function isUniformCard(card) {
|
|
2386
|
+
const separators = new Set(card.value.replace(/\d/g, ""));
|
|
2387
|
+
return separators.size <= 1;
|
|
2388
|
+
}
|
|
2389
|
+
function dropRescansInsideCards(spans, cards) {
|
|
2390
|
+
if (cards.length === 0 || !spans.some((span) => rescanTag.has(span))) {
|
|
2391
|
+
return spans;
|
|
2392
|
+
}
|
|
2393
|
+
const sorted = cards.filter(isUniformCard).sort((a, b) => a.start - b.start);
|
|
2394
|
+
const reach = [];
|
|
2395
|
+
let farthest = 0;
|
|
2396
|
+
for (const card of sorted) {
|
|
2397
|
+
farthest = Math.max(farthest, card.end);
|
|
2398
|
+
reach.push(farthest);
|
|
2399
|
+
}
|
|
2400
|
+
return spans.filter((span) => {
|
|
2401
|
+
if (!rescanTag.has(span)) {
|
|
2402
|
+
return true;
|
|
2403
|
+
}
|
|
2404
|
+
let low = 0;
|
|
2405
|
+
let high = sorted.length - 1;
|
|
2406
|
+
while (low <= high) {
|
|
2407
|
+
const mid = low + high >> 1;
|
|
2408
|
+
if (sorted[mid].start < span.end) {
|
|
2409
|
+
low = mid + 1;
|
|
2410
|
+
} else {
|
|
2411
|
+
high = mid - 1;
|
|
2412
|
+
}
|
|
2413
|
+
}
|
|
2414
|
+
return high < 0 || reach[high] <= span.start;
|
|
2415
|
+
});
|
|
2416
|
+
}
|
|
2417
|
+
function viewsOf(content, nonAscii) {
|
|
2418
|
+
const views = [];
|
|
2419
|
+
if (nonAscii) {
|
|
2420
|
+
views.push({});
|
|
2421
|
+
if (FULLWIDTH_CAPITAL.test(content)) {
|
|
2422
|
+
views.push({ letters: true });
|
|
2423
|
+
}
|
|
2424
|
+
if (SOFT_HYPHEN.test(content)) {
|
|
2425
|
+
views.push({ shyHyphen: true });
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
if (NUMBER_BREAK.test(content)) {
|
|
2429
|
+
views.push({ breaks: true });
|
|
2430
|
+
}
|
|
2431
|
+
return views;
|
|
2432
|
+
}
|
|
2433
|
+
function detectPii(content) {
|
|
2434
|
+
const perRule = scanRules(content);
|
|
2435
|
+
const nonAscii = NON_ASCII.test(content);
|
|
2436
|
+
if (nonAscii) {
|
|
2437
|
+
const emailRule = RULES[0];
|
|
2438
|
+
const ascii = scanRule(content, { ...emailRule, regex: asciiEmailScanner });
|
|
2439
|
+
const unicode = perRule[0];
|
|
2440
|
+
const extraEmails = ascii.filter((match) => !containedIn(unicode, match.start, match.end));
|
|
2441
|
+
if (extraEmails.length > 0) {
|
|
2442
|
+
perRule[0] = [...unicode, ...extraEmails].sort((x, y) => x.start - y.start || y.end - x.end);
|
|
2443
|
+
}
|
|
2444
|
+
}
|
|
2445
|
+
const views = viewsOf(content, nonAscii);
|
|
2446
|
+
for (const options of views) {
|
|
2447
|
+
for (let from = 0;from < content.length; ) {
|
|
2448
|
+
let start = from;
|
|
2449
|
+
let to = Math.min(content.length, from + NORMALIZE_CHUNK);
|
|
2450
|
+
if (start > 0 && (content.charCodeAt(start) & 64512) === 56320) {
|
|
2451
|
+
start -= 1;
|
|
2452
|
+
}
|
|
2453
|
+
if (to < content.length && (content.charCodeAt(to - 1) & 64512) === 55296) {
|
|
2454
|
+
to += 1;
|
|
2455
|
+
}
|
|
2456
|
+
const window = normalizeWindow(content, start, to, options);
|
|
2457
|
+
if (window !== null) {
|
|
2458
|
+
const extra = scanRules(window.text);
|
|
2459
|
+
extra.forEach((found, index) => {
|
|
2460
|
+
if (options.breaks && !NUMBER_RULES.has(index)) {
|
|
2461
|
+
return;
|
|
2462
|
+
}
|
|
2463
|
+
const numbers = NUMBER_RULES.has(index);
|
|
2464
|
+
const base = perRule[index];
|
|
2465
|
+
const added = [];
|
|
2466
|
+
for (const match of found) {
|
|
2467
|
+
const [a, b] = toOriginal(window, content, match.start, match.end);
|
|
2468
|
+
if (!numbers && b - a !== match.end - match.start) {
|
|
2469
|
+
continue;
|
|
2470
|
+
}
|
|
2471
|
+
if (start > 0 && a < start + NORMALIZE_SKIP) {
|
|
2472
|
+
continue;
|
|
2473
|
+
}
|
|
2474
|
+
if (!containedIn(base, a, b) && !containedIn(added, a, b)) {
|
|
2475
|
+
const copy = { ...match, start: a, end: b, value: content.slice(a, b) };
|
|
2476
|
+
if (rescanTag.has(match)) {
|
|
2477
|
+
rescanTag.add(copy);
|
|
2478
|
+
}
|
|
2479
|
+
added.push(copy);
|
|
2480
|
+
}
|
|
2481
|
+
}
|
|
2482
|
+
if (added.length > 0) {
|
|
2483
|
+
perRule[index] = coalesce(content, [...base, ...added].sort((x, y) => x.start - y.start || y.end - x.end));
|
|
2484
|
+
}
|
|
2485
|
+
});
|
|
2486
|
+
}
|
|
2487
|
+
if (to >= content.length) {
|
|
2488
|
+
break;
|
|
2489
|
+
}
|
|
2490
|
+
from = to - NORMALIZE_OVERLAP;
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
perRule[PHONE_RULE] = dropRescansInsideCards(perRule[PHONE_RULE], perRule[CARD_RULE]);
|
|
2494
|
+
return perRule.flat();
|
|
2495
|
+
}
|
|
2496
|
+
var UNICODE_EMAIL, ASCII_EMAIL, emailScanner, asciiEmailScanner, PHONE_FROM, PHONE_RUN_CHAR, PHONE_START_BLOCKER, DIGIT, PHONE_DIAL_GROUP, PHONE_FIRST_GROUP, PHONE_SEPARATOR, PHONE_NANP_SPACED, WHITESPACE, LINE_BREAK, PHONE_MAX_DIGITS = 15, PHONE_MIN_DIGITS = 9, rescanned, PHONE_JOINED_PIECE, PHONE_CARD_SHAPE, PHONE_GLUE, PHONE_GLUE_LETTER, PHONE_GLUE_EVIDENCE, phoneGlueScanner, phoneScanner, IBAN_START, IBAN_MAX_GROUPS = 8, IBAN_STATES, ibanFailed, ibanStamp = 0, ibanScanner, IBAN_LENGTHS, IBAN_GLUE_START, ibanGluedScanner, CARD_SEPARATOR = "[ \\t\\r\\n._/-]", CARD_EXTRA, CARD_GROUP_SEPARATOR, GLUE_LETTER_BEFORE, GLUE_LETTER_AFTER, NON_HEX_LETTER, GLUE_FLANK = 8, UUID_ANYWHERE, UUID_REACH = 36, SSN_BREAK_EXTRA, IPV4_EXTRA, RULES, CALENDAR_DATE, IDENTIFIER_CHAR, TOKEN_SCAN_LIMIT = 64, UUID_TOKEN, rescanTag, NORMALIZE_CHUNK, NORMALIZE_OVERLAP = 512, NORMALIZE_SKIP = 256, NON_ASCII, DECIMAL_DIGIT, digitValues, BREAK_CHAR, PHONE_RULE, CARD_RULE, NUMBER_RULES, FULLWIDTH_CAPITAL, SOFT_HYPHEN, NUMBER_BREAK;
|
|
2497
|
+
var init_pii = __esm(() => {
|
|
2498
|
+
UNICODE_EMAIL = {
|
|
2499
|
+
from: /(?:(?<=[\p{L}\p{N}\p{M}_])(?![\p{L}\p{N}\p{M}_])|(?<![\p{L}\p{N}\p{M}_])(?=[\p{L}\p{N}\p{M}_]))[\p{L}\p{N}\p{M}._%+-]+@[\p{L}\p{N}\p{M}.-]+\.[\p{L}\p{M}]{2,}(?![\p{L}\p{N}\p{M}_])/uy,
|
|
2500
|
+
localChar: /[\p{L}\p{N}\p{M}._%+-]/u,
|
|
2501
|
+
wordChar: /[\p{L}\p{N}\p{M}_]/u
|
|
2502
|
+
};
|
|
2503
|
+
ASCII_EMAIL = {
|
|
2504
|
+
from: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/y,
|
|
2505
|
+
localChar: /[A-Za-z0-9._%+-]/,
|
|
2506
|
+
wordChar: /\w/
|
|
2507
|
+
};
|
|
2508
|
+
emailScanner = emailScannerFor(UNICODE_EMAIL);
|
|
2509
|
+
asciiEmailScanner = emailScannerFor(ASCII_EMAIL);
|
|
2510
|
+
PHONE_FROM = /(?<![\w.])(\+?\d[\d\s().-]{7,}\d)(?!\w|\.\w)/y;
|
|
2511
|
+
PHONE_RUN_CHAR = /[\d\s().-]/;
|
|
2512
|
+
PHONE_START_BLOCKER = /[\w.]/;
|
|
2513
|
+
DIGIT = /\d/;
|
|
2514
|
+
PHONE_DIAL_GROUP = /^\(?\d{2,4}\)?(?:[-.]\d{2,4})*$/;
|
|
2515
|
+
PHONE_FIRST_GROUP = /^\+?\d{1,4}$/;
|
|
2516
|
+
PHONE_SEPARATOR = /[+()-]/;
|
|
2517
|
+
PHONE_NANP_SPACED = /^(?:\d{1,3} )?\d{3} \d{3} \d{4}$/;
|
|
2518
|
+
WHITESPACE = /\s/;
|
|
2519
|
+
LINE_BREAK = /[\n\r\u2028\u2029]/;
|
|
2520
|
+
rescanned = new WeakSet;
|
|
2521
|
+
PHONE_JOINED_PIECE = /^\+?\d{1,4}(?:[-.]\d{1,4})+$/;
|
|
2522
|
+
PHONE_CARD_SHAPE = /^\d{4}(?:[-.]\d{4})+$/;
|
|
2523
|
+
PHONE_GLUE = /(?<![\d.])\+?\d[\d().-]{7,}\d/g;
|
|
2524
|
+
PHONE_GLUE_LETTER = /[A-Za-z_]/;
|
|
2525
|
+
PHONE_GLUE_EVIDENCE = /[+()-]|\./;
|
|
2526
|
+
phoneGlueScanner = {
|
|
2527
|
+
lastIndex: 0,
|
|
2528
|
+
exec(content) {
|
|
2529
|
+
PHONE_GLUE.lastIndex = this.lastIndex;
|
|
2530
|
+
let m;
|
|
2531
|
+
while ((m = PHONE_GLUE.exec(content)) !== null) {
|
|
2532
|
+
const start = m.index;
|
|
2533
|
+
const end = start + m[0].length;
|
|
2534
|
+
const glued = start > 0 && PHONE_GLUE_LETTER.test(content[start - 1]) || end < content.length && PHONE_GLUE_LETTER.test(content[end]);
|
|
2535
|
+
if (glued && PHONE_GLUE_EVIDENCE.test(m[0]) && phoneVerdict(content, start, end)) {
|
|
2536
|
+
this.lastIndex = end;
|
|
2537
|
+
return { index: start, 0: m[0], 1: m[0] };
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
this.lastIndex = 0;
|
|
2541
|
+
return null;
|
|
2542
|
+
}
|
|
2543
|
+
};
|
|
2544
|
+
phoneScanner = {
|
|
2545
|
+
lastIndex: 0,
|
|
2546
|
+
pending: [],
|
|
2547
|
+
cursor: 0,
|
|
2548
|
+
runEnd: -1,
|
|
2549
|
+
runBaseline: -1,
|
|
2550
|
+
tailEnd: 0,
|
|
2551
|
+
reset() {
|
|
2552
|
+
this.lastIndex = 0;
|
|
2553
|
+
this.pending = [];
|
|
2554
|
+
this.cursor = 0;
|
|
2555
|
+
this.runEnd = -1;
|
|
2556
|
+
this.runBaseline = -1;
|
|
2557
|
+
this.tailEnd = 0;
|
|
2558
|
+
},
|
|
2559
|
+
tail(content) {
|
|
2560
|
+
for (let pos = this.lastIndex;pos < this.tailEnd; pos += 1) {
|
|
2561
|
+
if (!DIGIT.test(content[pos]) || PHONE_START_BLOCKER.test(content[pos - 1])) {
|
|
1833
2562
|
continue;
|
|
1834
2563
|
}
|
|
1835
|
-
if (
|
|
1836
|
-
|
|
2564
|
+
if (this.tailEnd - pos < 9) {
|
|
2565
|
+
break;
|
|
1837
2566
|
}
|
|
1838
|
-
|
|
1839
|
-
|
|
2567
|
+
const value = content.slice(pos, this.tailEnd);
|
|
2568
|
+
return { index: pos, 0: value, 1: value };
|
|
2569
|
+
}
|
|
2570
|
+
this.lastIndex = this.tailEnd;
|
|
2571
|
+
this.tailEnd = 0;
|
|
2572
|
+
return null;
|
|
2573
|
+
},
|
|
2574
|
+
prefix(content, start, end) {
|
|
2575
|
+
if (this.runEnd !== end) {
|
|
2576
|
+
this.runEnd = end;
|
|
2577
|
+
this.runBaseline = phoneBaselineEnd(content, end);
|
|
2578
|
+
}
|
|
2579
|
+
let digits = 0;
|
|
2580
|
+
let limit = end;
|
|
2581
|
+
const ends = [];
|
|
2582
|
+
for (let i = start;i < end; i += 1) {
|
|
2583
|
+
const ch = content[i];
|
|
2584
|
+
if (DIGIT.test(ch)) {
|
|
2585
|
+
digits += 1;
|
|
2586
|
+
if (digits > PHONE_MAX_DIGITS) {
|
|
2587
|
+
limit = i;
|
|
2588
|
+
break;
|
|
2589
|
+
}
|
|
2590
|
+
} else if (WHITESPACE.test(ch) && DIGIT.test(content[i - 1]) && digits >= PHONE_MIN_DIGITS) {
|
|
2591
|
+
ends.push(i);
|
|
1840
2592
|
}
|
|
1841
|
-
|
|
1842
|
-
|
|
2593
|
+
}
|
|
2594
|
+
const baseline = this.runBaseline;
|
|
2595
|
+
if (baseline < end && baseline >= start + 9 && baseline <= limit && phoneVerdict(content, start, baseline)) {
|
|
2596
|
+
return { end: baseline, baseline: true };
|
|
2597
|
+
}
|
|
2598
|
+
for (let k = ends.length - 1;k >= 0; k -= 1) {
|
|
2599
|
+
const e = ends[k];
|
|
2600
|
+
const value = content.slice(start, e);
|
|
2601
|
+
if (e !== baseline && hasPhoneEvidence(value, !WHITESPACE.test(value)) && phoneVerdict(content, start, e)) {
|
|
2602
|
+
return { end: e, baseline: false };
|
|
1843
2603
|
}
|
|
1844
2604
|
}
|
|
1845
|
-
|
|
1846
|
-
|
|
2605
|
+
return null;
|
|
2606
|
+
},
|
|
2607
|
+
exec(content) {
|
|
2608
|
+
for (;; ) {
|
|
2609
|
+
if (this.cursor < this.pending.length) {
|
|
2610
|
+
return this.pending[this.cursor++];
|
|
2611
|
+
}
|
|
2612
|
+
this.pending = [];
|
|
2613
|
+
this.cursor = 0;
|
|
2614
|
+
let m;
|
|
2615
|
+
if (this.tailEnd > this.lastIndex) {
|
|
2616
|
+
m = this.tail(content);
|
|
2617
|
+
if (m === null) {
|
|
2618
|
+
continue;
|
|
2619
|
+
}
|
|
2620
|
+
} else {
|
|
2621
|
+
m = execPhone(content, this.lastIndex);
|
|
2622
|
+
if (m === null) {
|
|
2623
|
+
this.lastIndex = 0;
|
|
2624
|
+
return null;
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
const end = m.index + (m[0] ?? "").length;
|
|
2628
|
+
this.lastIndex = end;
|
|
2629
|
+
this.tailEnd = 0;
|
|
2630
|
+
if (phoneVerdict(content, m.index, end)) {
|
|
2631
|
+
return m;
|
|
2632
|
+
}
|
|
2633
|
+
const prefix = this.prefix(content, m.index, end);
|
|
2634
|
+
if (prefix !== null) {
|
|
2635
|
+
const value = content.slice(m.index, prefix.end);
|
|
2636
|
+
const found = { index: m.index, 0: value, 1: value };
|
|
2637
|
+
if (!prefix.baseline) {
|
|
2638
|
+
rescanned.add(found);
|
|
2639
|
+
}
|
|
2640
|
+
this.lastIndex = prefix.end;
|
|
2641
|
+
this.tailEnd = end;
|
|
2642
|
+
return found;
|
|
2643
|
+
}
|
|
2644
|
+
this.pending = withJoined(joinedPhones(content, m.index, end), rescanPhoneRun(content, m.index, end));
|
|
2645
|
+
for (const found of this.pending) {
|
|
2646
|
+
rescanned.add(found);
|
|
2647
|
+
}
|
|
1847
2648
|
}
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
2649
|
+
}
|
|
2650
|
+
};
|
|
2651
|
+
IBAN_START = /(?<!\w)[A-Z]{2}\d{2}/g;
|
|
2652
|
+
IBAN_STATES = (4 + IBAN_MAX_GROUPS * 5 + 1) * (IBAN_MAX_GROUPS + 1);
|
|
2653
|
+
ibanFailed = new Uint32Array(IBAN_STATES);
|
|
2654
|
+
ibanScanner = {
|
|
2655
|
+
lastIndex: 0,
|
|
2656
|
+
exec(content) {
|
|
2657
|
+
IBAN_START.lastIndex = this.lastIndex;
|
|
2658
|
+
let m;
|
|
2659
|
+
while ((m = IBAN_START.exec(content)) !== null) {
|
|
2660
|
+
ibanStamp += 1;
|
|
2661
|
+
if (ibanStamp === 4294967295) {
|
|
2662
|
+
ibanFailed.fill(0);
|
|
2663
|
+
ibanStamp = 1;
|
|
2664
|
+
}
|
|
2665
|
+
const end = ibanEnd(content, m.index, m.index + 4, 0);
|
|
2666
|
+
if (end >= 0) {
|
|
2667
|
+
this.lastIndex = end;
|
|
2668
|
+
return { index: m.index, 0: content.slice(m.index, end) };
|
|
2669
|
+
}
|
|
2670
|
+
IBAN_START.lastIndex = m.index + 1;
|
|
1862
2671
|
}
|
|
2672
|
+
this.lastIndex = 0;
|
|
2673
|
+
return null;
|
|
1863
2674
|
}
|
|
1864
|
-
}
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
2675
|
+
};
|
|
2676
|
+
IBAN_LENGTHS = {
|
|
2677
|
+
AD: 24,
|
|
2678
|
+
AE: 23,
|
|
2679
|
+
AL: 28,
|
|
2680
|
+
AT: 20,
|
|
2681
|
+
AZ: 28,
|
|
2682
|
+
BA: 20,
|
|
2683
|
+
BE: 16,
|
|
2684
|
+
BG: 22,
|
|
2685
|
+
BH: 22,
|
|
2686
|
+
BR: 29,
|
|
2687
|
+
BY: 28,
|
|
2688
|
+
CH: 21,
|
|
2689
|
+
CR: 22,
|
|
2690
|
+
CY: 28,
|
|
2691
|
+
CZ: 24,
|
|
2692
|
+
DE: 22,
|
|
2693
|
+
DK: 18,
|
|
2694
|
+
DO: 28,
|
|
2695
|
+
EE: 20,
|
|
2696
|
+
EG: 29,
|
|
2697
|
+
ES: 24,
|
|
2698
|
+
FI: 18,
|
|
2699
|
+
FO: 18,
|
|
2700
|
+
FR: 27,
|
|
2701
|
+
GB: 22,
|
|
2702
|
+
GE: 22,
|
|
2703
|
+
GI: 23,
|
|
2704
|
+
GL: 18,
|
|
2705
|
+
GR: 27,
|
|
2706
|
+
GT: 28,
|
|
2707
|
+
HR: 21,
|
|
2708
|
+
HU: 28,
|
|
2709
|
+
IE: 22,
|
|
2710
|
+
IL: 23,
|
|
2711
|
+
IQ: 23,
|
|
2712
|
+
IS: 26,
|
|
2713
|
+
IT: 27,
|
|
2714
|
+
JO: 30,
|
|
2715
|
+
KW: 30,
|
|
2716
|
+
KZ: 20,
|
|
2717
|
+
LB: 28,
|
|
2718
|
+
LC: 32,
|
|
2719
|
+
LI: 21,
|
|
2720
|
+
LT: 20,
|
|
2721
|
+
LU: 20,
|
|
2722
|
+
LV: 21,
|
|
2723
|
+
MC: 27,
|
|
2724
|
+
MD: 24,
|
|
2725
|
+
ME: 22,
|
|
2726
|
+
MK: 19,
|
|
2727
|
+
MR: 27,
|
|
2728
|
+
MT: 31,
|
|
2729
|
+
MU: 30,
|
|
2730
|
+
NL: 18,
|
|
2731
|
+
NO: 15,
|
|
2732
|
+
PK: 24,
|
|
2733
|
+
PL: 28,
|
|
2734
|
+
PS: 29,
|
|
2735
|
+
PT: 25,
|
|
2736
|
+
QA: 29,
|
|
2737
|
+
RO: 24,
|
|
2738
|
+
RS: 22,
|
|
2739
|
+
SA: 24,
|
|
2740
|
+
SC: 31,
|
|
2741
|
+
SE: 24,
|
|
2742
|
+
SI: 19,
|
|
2743
|
+
SK: 24,
|
|
2744
|
+
SM: 27,
|
|
2745
|
+
ST: 25,
|
|
2746
|
+
SV: 28,
|
|
2747
|
+
TL: 23,
|
|
2748
|
+
TN: 24,
|
|
2749
|
+
TR: 26,
|
|
2750
|
+
UA: 29,
|
|
2751
|
+
VA: 22,
|
|
2752
|
+
VG: 24,
|
|
2753
|
+
XK: 20
|
|
2754
|
+
};
|
|
2755
|
+
IBAN_GLUE_START = /[A-Z]{2}\d{2}/g;
|
|
2756
|
+
ibanGluedScanner = {
|
|
2757
|
+
lastIndex: 0,
|
|
2758
|
+
exec(content) {
|
|
2759
|
+
IBAN_GLUE_START.lastIndex = this.lastIndex;
|
|
2760
|
+
let m;
|
|
2761
|
+
while ((m = IBAN_GLUE_START.exec(content)) !== null) {
|
|
2762
|
+
const length = IBAN_LENGTHS[m[0].slice(0, 2)];
|
|
2763
|
+
if (length !== undefined) {
|
|
2764
|
+
let count = 0;
|
|
2765
|
+
let i = m.index;
|
|
2766
|
+
while (i < content.length && count < length) {
|
|
2767
|
+
const code = content.charCodeAt(i);
|
|
2768
|
+
if (isIbanChar(code)) {
|
|
2769
|
+
count += 1;
|
|
2770
|
+
} else if (!(code === 32 && count >= 4 && isIbanChar(content.charCodeAt(i + 1)))) {
|
|
2771
|
+
break;
|
|
2772
|
+
}
|
|
2773
|
+
i += 1;
|
|
2774
|
+
}
|
|
2775
|
+
if (count === length) {
|
|
2776
|
+
this.lastIndex = i;
|
|
2777
|
+
return { index: m.index, 0: content.slice(m.index, i) };
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
IBAN_GLUE_START.lastIndex = m.index + 1;
|
|
2781
|
+
}
|
|
2782
|
+
this.lastIndex = 0;
|
|
2783
|
+
return null;
|
|
2784
|
+
}
|
|
2785
|
+
};
|
|
2786
|
+
CARD_EXTRA = new RegExp(`(?<!\\d${CARD_SEPARATOR}{0,2})[2-6](?:${CARD_SEPARATOR}{0,2}\\d){12,18}(?!\\d)(?!${CARD_SEPARATOR}{1,2}\\d)`, "g");
|
|
2787
|
+
CARD_GROUP_SEPARATOR = /[^\d]+/g;
|
|
2788
|
+
GLUE_LETTER_BEFORE = /[A-Za-z_]+$/;
|
|
2789
|
+
GLUE_LETTER_AFTER = /^[A-Za-z_]+/;
|
|
2790
|
+
NON_HEX_LETTER = /[G-Zg-z_]/;
|
|
2791
|
+
UUID_ANYWHERE = /[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}/g;
|
|
2792
|
+
SSN_BREAK_EXTRA = /(?<![0-9])\d{3}(?:\r\n|[\t\r\n])\d{2}(?:\r\n|[\t\r\n])\d{4}(?![0-9])/g;
|
|
2793
|
+
IPV4_EXTRA = /(?<![0-9.])(?<!\b[vV])\d{1,3}(?:[\t\r\n]{0,2}\.[\t\r\n]{0,2}\d{1,3}){3}(?![0-9])/g;
|
|
1869
2794
|
RULES = [
|
|
1870
2795
|
{
|
|
1871
2796
|
policyId: "pii.email",
|
|
1872
2797
|
mask: "email",
|
|
1873
|
-
regex:
|
|
2798
|
+
regex: emailScanner,
|
|
1874
2799
|
severity: "medium",
|
|
1875
2800
|
confidence: 0.85
|
|
1876
2801
|
},
|
|
1877
2802
|
{
|
|
1878
2803
|
policyId: "pii.phone",
|
|
1879
2804
|
mask: "phone",
|
|
1880
|
-
regex:
|
|
2805
|
+
regex: phoneScanner,
|
|
1881
2806
|
severity: "medium",
|
|
1882
2807
|
confidence: 0.7,
|
|
1883
|
-
valueGroup: 1
|
|
2808
|
+
valueGroup: 1,
|
|
2809
|
+
also: [{ regex: phoneGlueScanner }]
|
|
1884
2810
|
},
|
|
1885
2811
|
{
|
|
1886
2812
|
policyId: "pii.address",
|
|
@@ -1892,7 +2818,7 @@ var init_pii = __esm(() => {
|
|
|
1892
2818
|
{
|
|
1893
2819
|
policyId: "pii.person-name",
|
|
1894
2820
|
mask: "name",
|
|
1895
|
-
regex: /\b(?:name\s+is|customer|client|user|patient|employee)
|
|
2821
|
+
regex: /\b(?:name\s+is|customer|client|user|patient|employee)(?:\s*:)?\s+([A-Z][a-z]+\s+[A-Z][a-z]+)\b/g,
|
|
1896
2822
|
severity: "low",
|
|
1897
2823
|
confidence: 0.45,
|
|
1898
2824
|
valueGroup: 1
|
|
@@ -1900,10 +2826,11 @@ var init_pii = __esm(() => {
|
|
|
1900
2826
|
{
|
|
1901
2827
|
policyId: "pii.iban",
|
|
1902
2828
|
mask: "iban",
|
|
1903
|
-
regex:
|
|
2829
|
+
regex: ibanScanner,
|
|
1904
2830
|
severity: "high",
|
|
1905
2831
|
confidence: 0.9,
|
|
1906
|
-
validate: isValidIban
|
|
2832
|
+
validate: isValidIban,
|
|
2833
|
+
also: [{ regex: ibanGluedScanner }]
|
|
1907
2834
|
},
|
|
1908
2835
|
{
|
|
1909
2836
|
policyId: "pii.credit-card",
|
|
@@ -1911,15 +2838,24 @@ var init_pii = __esm(() => {
|
|
|
1911
2838
|
regex: /\b\d(?:[ -]?\d){12,18}\b/g,
|
|
1912
2839
|
severity: "high",
|
|
1913
2840
|
confidence: 0.9,
|
|
1914
|
-
validate: isValidCreditCard
|
|
2841
|
+
validate: isValidCreditCard,
|
|
2842
|
+
accept: outsideUuid,
|
|
2843
|
+
also: [
|
|
2844
|
+
{
|
|
2845
|
+
regex: CARD_EXTRA,
|
|
2846
|
+
validate: (value) => isCardCandidate(value) && isValidCreditCard(value.replace(/\D/g, "")),
|
|
2847
|
+
accept: hasNonHexGlue
|
|
2848
|
+
}
|
|
2849
|
+
]
|
|
1915
2850
|
},
|
|
1916
2851
|
{
|
|
1917
2852
|
policyId: "pii.ssn",
|
|
1918
2853
|
mask: "ssn",
|
|
1919
|
-
regex:
|
|
2854
|
+
regex: /(?<![0-9])\d{3}-\d{2}-\d{4}(?![0-9])/g,
|
|
1920
2855
|
severity: "high",
|
|
1921
2856
|
confidence: 0.85,
|
|
1922
|
-
validate: isValidSsn
|
|
2857
|
+
validate: isValidSsn,
|
|
2858
|
+
also: [{ regex: SSN_BREAK_EXTRA, validate: (value) => isValidSsn(value.replace(/\r\n|[\t\r\n]/g, "-")) }]
|
|
1923
2859
|
},
|
|
1924
2860
|
{
|
|
1925
2861
|
policyId: "pii.ip",
|
|
@@ -1927,12 +2863,25 @@ var init_pii = __esm(() => {
|
|
|
1927
2863
|
regex: /\b(?:(?:\d{1,3}\.){3}\d{1,3}|(?:[0-9A-Fa-f]{1,4}:){2,7}[0-9A-Fa-f]{1,4}|::1)\b/g,
|
|
1928
2864
|
severity: "low",
|
|
1929
2865
|
confidence: 0.6,
|
|
1930
|
-
validate: isValidIp
|
|
2866
|
+
validate: isValidIp,
|
|
2867
|
+
also: [{ regex: IPV4_EXTRA, validate: (value) => isValidIp(value.replace(/[\t\r\n]/g, "")) }]
|
|
1931
2868
|
}
|
|
1932
2869
|
];
|
|
1933
2870
|
CALENDAR_DATE = /(?:19|20)\d{2}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])/;
|
|
1934
2871
|
IDENTIFIER_CHAR = /[0-9A-Za-z_-]/;
|
|
1935
2872
|
UUID_TOKEN = /^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$/;
|
|
2873
|
+
rescanTag = new WeakSet;
|
|
2874
|
+
NORMALIZE_CHUNK = 1 << 21;
|
|
2875
|
+
NON_ASCII = /[\u0080-\uffff]/;
|
|
2876
|
+
DECIMAL_DIGIT = /\p{Nd}/u;
|
|
2877
|
+
digitValues = new Map;
|
|
2878
|
+
BREAK_CHAR = /[\t\r\n]/;
|
|
2879
|
+
PHONE_RULE = RULES.findIndex((rule) => rule.policyId === "pii.phone");
|
|
2880
|
+
CARD_RULE = RULES.findIndex((rule) => rule.policyId === "pii.credit-card");
|
|
2881
|
+
NUMBER_RULES = new Set(RULES.flatMap((rule, index) => ["pii.phone", "pii.iban", "pii.credit-card", "pii.ssn", "pii.ip"].includes(rule.policyId) ? [index] : []));
|
|
2882
|
+
FULLWIDTH_CAPITAL = /[\uff21-\uff3a]/;
|
|
2883
|
+
SOFT_HYPHEN = /\u00ad/;
|
|
2884
|
+
NUMBER_BREAK = /[-.][\t\r\n]|[\t\r\n][-.]/;
|
|
1936
2885
|
});
|
|
1937
2886
|
|
|
1938
2887
|
// src/security/detect/secrets.ts
|
|
@@ -4348,17 +5297,20 @@ function locationFor(content, match) {
|
|
|
4348
5297
|
};
|
|
4349
5298
|
}
|
|
4350
5299
|
function applyRedaction(content, matches) {
|
|
4351
|
-
const redactable = matches.filter((m) => m.mask !== undefined).sort((a, b) => a.start - b.start);
|
|
5300
|
+
const redactable = matches.filter((m) => m.mask !== undefined).sort((a, b) => a.start - b.start || b.end - a.end);
|
|
4352
5301
|
let out = "";
|
|
4353
5302
|
let cursor = 0;
|
|
4354
5303
|
for (const match of redactable) {
|
|
4355
|
-
|
|
4356
|
-
if (start >= match.end) {
|
|
5304
|
+
if (match.end <= cursor) {
|
|
4357
5305
|
continue;
|
|
4358
5306
|
}
|
|
4359
|
-
|
|
5307
|
+
if (match.start < cursor) {
|
|
5308
|
+
cursor = match.end;
|
|
5309
|
+
continue;
|
|
5310
|
+
}
|
|
5311
|
+
out += content.slice(cursor, match.start);
|
|
4360
5312
|
out += maskFor(match);
|
|
4361
|
-
cursor =
|
|
5313
|
+
cursor = match.end;
|
|
4362
5314
|
}
|
|
4363
5315
|
out += content.slice(cursor);
|
|
4364
5316
|
return out;
|
|
@@ -20145,7 +21097,7 @@ var package_default;
|
|
|
20145
21097
|
var init_package = __esm(() => {
|
|
20146
21098
|
package_default = {
|
|
20147
21099
|
name: "@mrciphersmith/keryx",
|
|
20148
|
-
version: "0.3.
|
|
21100
|
+
version: "0.3.78",
|
|
20149
21101
|
description: "Version-controlled project context for AI coding agents: code graph, architecture wiki, project memory, relevant tests, quality signals, and task flows.",
|
|
20150
21102
|
private: false,
|
|
20151
21103
|
publishConfig: {
|
|
@@ -20223,10 +21175,10 @@ var init_package = __esm(() => {
|
|
|
20223
21175
|
qs: "^6.16.0",
|
|
20224
21176
|
"@hono/node-server": "^1.19.15",
|
|
20225
21177
|
protobufjs: "7.6.6",
|
|
20226
|
-
sharp: "0.35.
|
|
21178
|
+
sharp: "0.35.5"
|
|
20227
21179
|
},
|
|
20228
21180
|
optionalDependencies: {
|
|
20229
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
21181
|
+
"@modelcontextprotocol/sdk": "^1.32.1",
|
|
20230
21182
|
"@opentui/core": "^0.4.5",
|
|
20231
21183
|
"web-tree-sitter": "^0.22.6"
|
|
20232
21184
|
},
|
|
@@ -24868,6 +25820,7 @@ var init_service2 = __esm(() => {
|
|
|
24868
25820
|
init_path_scan();
|
|
24869
25821
|
init_read_source();
|
|
24870
25822
|
init_redact();
|
|
25823
|
+
init_pii();
|
|
24871
25824
|
init_secrets();
|
|
24872
25825
|
init_entropy();
|
|
24873
25826
|
init_incidents();
|
|
@@ -111336,6 +112289,22 @@ async function attemptJevRequest(fetchFn, apiKey, body, timeoutMs, external) {
|
|
|
111336
112289
|
}
|
|
111337
112290
|
}
|
|
111338
112291
|
}
|
|
112292
|
+
async function cancellableJevSleep(sleep, ms, signal, timeoutMs) {
|
|
112293
|
+
if (signal === undefined)
|
|
112294
|
+
return sleep(ms);
|
|
112295
|
+
if (signal.aborted)
|
|
112296
|
+
throw new JevTimeoutError(timeoutMs);
|
|
112297
|
+
let onAbort = () => {};
|
|
112298
|
+
const aborted = new Promise((_, reject) => {
|
|
112299
|
+
onAbort = () => reject(new JevTimeoutError(timeoutMs));
|
|
112300
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
112301
|
+
});
|
|
112302
|
+
try {
|
|
112303
|
+
await Promise.race([sleep(ms), aborted]);
|
|
112304
|
+
} finally {
|
|
112305
|
+
signal.removeEventListener("abort", onAbort);
|
|
112306
|
+
}
|
|
112307
|
+
}
|
|
111339
112308
|
async function callJevSystemOne(fetchFn = globalThis.fetch, input, opts) {
|
|
111340
112309
|
const externalSetting = await resolveExternalSetting({ cwd: opts?.cwd ?? process.cwd(), ...opts?.dir !== undefined ? { dir: opts.dir } : {} });
|
|
111341
112310
|
if (externalSetting.value === "off") {
|
|
@@ -111361,6 +112330,8 @@ async function callJevSystemOne(fetchFn = globalThis.fetch, input, opts) {
|
|
|
111361
112330
|
let text;
|
|
111362
112331
|
let attempt = 0;
|
|
111363
112332
|
for (;; ) {
|
|
112333
|
+
if (external?.aborted)
|
|
112334
|
+
throw new JevTimeoutError(timeoutMs);
|
|
111364
112335
|
attempt += 1;
|
|
111365
112336
|
const outcome = await attemptJevRequest(fetchFn, apiKey, body, timeoutMs, external);
|
|
111366
112337
|
if (outcome.kind === "timeout") {
|
|
@@ -111370,7 +112341,7 @@ async function callJevSystemOne(fetchFn = globalThis.fetch, input, opts) {
|
|
|
111370
112341
|
if (attempt >= JEV_MAX_ATTEMPTS) {
|
|
111371
112342
|
throw new JevNetworkError(attempt, outcome.cause);
|
|
111372
112343
|
}
|
|
111373
|
-
await sleepFn
|
|
112344
|
+
await cancellableJevSleep(sleepFn, jevBackoffMs(attempt), external, timeoutMs);
|
|
111374
112345
|
continue;
|
|
111375
112346
|
}
|
|
111376
112347
|
if (outcome.ok) {
|
|
@@ -111381,7 +112352,7 @@ async function callJevSystemOne(fetchFn = globalThis.fetch, input, opts) {
|
|
|
111381
112352
|
text = outcome.text;
|
|
111382
112353
|
if (JEV_RETRYABLE_STATUSES.has(status) && attempt < JEV_MAX_ATTEMPTS) {
|
|
111383
112354
|
const retryAfterMs = retryAfterMsFromHeader(outcome.retryAfterHeader);
|
|
111384
|
-
await sleepFn
|
|
112355
|
+
await cancellableJevSleep(sleepFn, retryAfterMs ?? jevBackoffMs(attempt), external, timeoutMs);
|
|
111385
112356
|
continue;
|
|
111386
112357
|
}
|
|
111387
112358
|
if (status === 401) {
|
|
@@ -126919,13 +127890,75 @@ var init_proxy = __esm(() => {
|
|
|
126919
127890
|
init_guard();
|
|
126920
127891
|
});
|
|
126921
127892
|
|
|
127893
|
+
// src/commands/granted-json-ids.ts
|
|
127894
|
+
function letters(index) {
|
|
127895
|
+
return String(index).replace(/\d/g, (d) => String.fromCharCode(97 + Number(d)));
|
|
127896
|
+
}
|
|
127897
|
+
function protectJsonIds(text) {
|
|
127898
|
+
const head = text.trim();
|
|
127899
|
+
if (head.length === 0 || head[0] !== "[" && head[0] !== "{")
|
|
127900
|
+
return;
|
|
127901
|
+
if (text.includes(PLACEHOLDER))
|
|
127902
|
+
return;
|
|
127903
|
+
try {
|
|
127904
|
+
JSON.parse(head);
|
|
127905
|
+
} catch {
|
|
127906
|
+
return;
|
|
127907
|
+
}
|
|
127908
|
+
const hidden = [];
|
|
127909
|
+
const hide = (value) => {
|
|
127910
|
+
hidden.push(value);
|
|
127911
|
+
return `${PLACEHOLDER}${letters(hidden.length - 1)}x`;
|
|
127912
|
+
};
|
|
127913
|
+
let out = "";
|
|
127914
|
+
let i = 0;
|
|
127915
|
+
while (i < text.length) {
|
|
127916
|
+
const ch = text[i];
|
|
127917
|
+
if (ch === '"') {
|
|
127918
|
+
let j = i + 1;
|
|
127919
|
+
while (j < text.length && text[j] !== '"')
|
|
127920
|
+
j += text[j] === "\\" ? 2 : 1;
|
|
127921
|
+
const literal = text.slice(i, j + 1);
|
|
127922
|
+
const inner = literal.slice(1, -1);
|
|
127923
|
+
const prefix = GITHUB_URL_PREFIX.exec(inner);
|
|
127924
|
+
const rest = prefix === null ? "" : inner.slice(prefix[0].length);
|
|
127925
|
+
out += prefix !== null && GITHUB_URL_REST.test(rest) ? `"${prefix[0]}${rest.replace(/\d+/g, hide)}"` : literal;
|
|
127926
|
+
i = j + 1;
|
|
127927
|
+
continue;
|
|
127928
|
+
}
|
|
127929
|
+
const number = ch === "-" || ch >= "0" && ch <= "9" ? NUMBER2.exec(text.slice(i, i + 40)) : null;
|
|
127930
|
+
if (number !== null) {
|
|
127931
|
+
const key = /"((?:[^"\\]|\\.)*)"\s*:\s*$/.exec(out.slice(-120))?.[1] ?? "";
|
|
127932
|
+
out += SENSITIVE_KEY.test(key) ? number[0] : hide(number[0]);
|
|
127933
|
+
i += number[0].length;
|
|
127934
|
+
continue;
|
|
127935
|
+
}
|
|
127936
|
+
out += ch;
|
|
127937
|
+
i += 1;
|
|
127938
|
+
}
|
|
127939
|
+
return {
|
|
127940
|
+
text: out,
|
|
127941
|
+
restore: (scrubbed) => scrubbed.replace(new RegExp(`${PLACEHOLDER}([a-j]+)x`, "g"), (whole, index) => {
|
|
127942
|
+
const at = Number(index.replace(/[a-j]/g, (c) => String(c.charCodeAt(0) - 97)));
|
|
127943
|
+
return hidden[at] ?? whole;
|
|
127944
|
+
})
|
|
127945
|
+
};
|
|
127946
|
+
}
|
|
127947
|
+
var PLACEHOLDER = "kxjsonid", GITHUB_URL_PREFIX, GITHUB_URL_REST, SENSITIVE_KEY, NUMBER2;
|
|
127948
|
+
var init_granted_json_ids = __esm(() => {
|
|
127949
|
+
GITHUB_URL_PREFIX = /^https:\/\/(?:api\.)?github\.com\/(?:repos\/)?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\//;
|
|
127950
|
+
GITHUB_URL_REST = /^(?:actions\/runs|pull|issues)\/\d+(?:\/[A-Za-z0-9_-]+)*(?:#[A-Za-z_-]+-?\d+)?$/;
|
|
127951
|
+
SENSITIVE_KEY = /token|secret|pass|key|auth|credential|private|ssn|phone|card|account|iban|pin|otp/i;
|
|
127952
|
+
NUMBER2 = /^-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/;
|
|
127953
|
+
});
|
|
127954
|
+
|
|
126922
127955
|
// src/commands/trigger-agent-task.ts
|
|
126923
127956
|
import { execFile as execFile11 } from "child_process";
|
|
126924
127957
|
import { randomUUID as randomUUID35 } from "crypto";
|
|
126925
127958
|
import { mkdtemp as mkdtemp5, mkdir as mkdir60, readdir as readdir42, rm as rm19, writeFile as writeFile37 } from "fs/promises";
|
|
126926
127959
|
import { homedir as homedir12, tmpdir as tmpdir11 } from "os";
|
|
126927
127960
|
import path266 from "path";
|
|
126928
|
-
function scrubGrantedOutput(text, env, secrets = []) {
|
|
127961
|
+
function scrubGrantedOutput(text, env, secrets = [], json = false) {
|
|
126929
127962
|
let out = text;
|
|
126930
127963
|
for (const value of secrets) {
|
|
126931
127964
|
if (value.length >= 8)
|
|
@@ -126938,10 +127971,18 @@ function scrubGrantedOutput(text, env, secrets = []) {
|
|
|
126938
127971
|
}
|
|
126939
127972
|
for (const [shape, mask] of TOKEN_SHAPES)
|
|
126940
127973
|
out = out.replace(shape, mask);
|
|
127974
|
+
if (json) {
|
|
127975
|
+
const at = out.indexOf(STDERR_MARK);
|
|
127976
|
+
const protectedJson = protectJsonIds(at === -1 ? out : out.slice(0, at));
|
|
127977
|
+
if (protectedJson !== undefined) {
|
|
127978
|
+
const tail = at === -1 ? "" : redactSensitiveText(out.slice(at));
|
|
127979
|
+
return protectedJson.restore(redactSensitiveText(protectedJson.text)) + tail;
|
|
127980
|
+
}
|
|
127981
|
+
}
|
|
126941
127982
|
return redactSensitiveText(out);
|
|
126942
127983
|
}
|
|
126943
|
-
function scrubThenCap(raw, env, secrets, max = GRANTED_MAX_BYTES) {
|
|
126944
|
-
const scrubbed = scrubGrantedOutput(raw, env, secrets);
|
|
127984
|
+
function scrubThenCap(raw, env, secrets, max = GRANTED_MAX_BYTES, json = false) {
|
|
127985
|
+
const scrubbed = scrubGrantedOutput(raw, env, secrets, json);
|
|
126945
127986
|
if (scrubbed.length <= max)
|
|
126946
127987
|
return scrubbed;
|
|
126947
127988
|
const head = scrubbed.slice(0, max);
|
|
@@ -126959,7 +128000,7 @@ ${stderr}` : ""}`;
|
|
|
126959
128000
|
const code = error === null ? 0 : typeof error.code === "number" ? error.code : null;
|
|
126960
128001
|
const reason = error === null ? "" : `
|
|
126961
128002
|
[exit: ${code ?? error.message}]`;
|
|
126962
|
-
resolve({ output: `${scrubThenCap(raw, env, secrets)}${scrubGrantedOutput(reason, env, secrets)}`, exitCode: code, ok: error === null });
|
|
128003
|
+
resolve({ output: `${scrubThenCap(raw, env, secrets, GRANTED_MAX_BYTES, true)}${scrubGrantedOutput(reason, env, secrets)}`, exitCode: code, ok: error === null });
|
|
126963
128004
|
});
|
|
126964
128005
|
});
|
|
126965
128006
|
}
|
|
@@ -127326,7 +128367,9 @@ function defaultArmTimeout2(ms, fire) {
|
|
|
127326
128367
|
timer.unref?.();
|
|
127327
128368
|
return () => clearTimeout(timer);
|
|
127328
128369
|
}
|
|
127329
|
-
var GRANTED_TIMEOUT_MS = 30000, GRANTED_MAX_BYTES,
|
|
128370
|
+
var GRANTED_TIMEOUT_MS = 30000, GRANTED_MAX_BYTES, STDERR_MARK = `
|
|
128371
|
+
[stderr]
|
|
128372
|
+
`, SECRET_ENV_NAME, TOKEN_SHAPES;
|
|
127330
128373
|
var init_trigger_agent_task = __esm(() => {
|
|
127331
128374
|
init_agent();
|
|
127332
128375
|
init_agent_hooks();
|
|
@@ -127338,6 +128381,7 @@ var init_trigger_agent_task = __esm(() => {
|
|
|
127338
128381
|
init_fs();
|
|
127339
128382
|
init_maintenance_lock();
|
|
127340
128383
|
init_service2();
|
|
128384
|
+
init_granted_json_ids();
|
|
127341
128385
|
init_granted_binary();
|
|
127342
128386
|
init_schedule_verify();
|
|
127343
128387
|
init_granted_tools();
|
|
@@ -127388,9 +128432,9 @@ function ghEnvForProject(projectRoot, base = process.env) {
|
|
|
127388
128432
|
function parseGrantedOutput(output, ok, exitCode) {
|
|
127389
128433
|
const exit = EXIT_MARK.exec(output);
|
|
127390
128434
|
const withoutExit = exit === null ? output : output.slice(0, exit.index);
|
|
127391
|
-
const stderrAt = withoutExit.indexOf(
|
|
128435
|
+
const stderrAt = withoutExit.indexOf(STDERR_MARK2);
|
|
127392
128436
|
const stdout = stderrAt === -1 ? withoutExit : withoutExit.slice(0, stderrAt);
|
|
127393
|
-
const stderr = stderrAt === -1 ? "" : withoutExit.slice(stderrAt +
|
|
128437
|
+
const stderr = stderrAt === -1 ? "" : withoutExit.slice(stderrAt + STDERR_MARK2.length).trim();
|
|
127394
128438
|
if (!ok) {
|
|
127395
128439
|
const why = stderr.length > 0 ? stderr : exit !== null ? exit[0].trim() : "gh failed";
|
|
127396
128440
|
return { ok: false, stdout: "", detail: `${why.replace(/\s+/g, " ").slice(0, 300)}`, exitCode };
|
|
@@ -127400,7 +128444,7 @@ function parseGrantedOutput(output, ok, exitCode) {
|
|
|
127400
128444
|
}
|
|
127401
128445
|
return { ok: true, stdout, detail: "", exitCode };
|
|
127402
128446
|
}
|
|
127403
|
-
var GH_ENV_ALLOWED,
|
|
128447
|
+
var GH_ENV_ALLOWED, STDERR_MARK2 = `
|
|
127404
128448
|
[stderr]
|
|
127405
128449
|
`, TRUNCATED, EXIT_MARK, defaultGhRunner = async (call) => {
|
|
127406
128450
|
const result = await runGrantedCommand(call.bin, call.argv, call.cwd, call.env, call.signal, call.secrets);
|
|
@@ -144239,52 +145283,77 @@ class NullClassifier {
|
|
|
144239
145283
|
}
|
|
144240
145284
|
|
|
144241
145285
|
// src/harness/decision/classify-turn.ts
|
|
144242
|
-
async function withTimeout4(source, run, timeoutMs) {
|
|
145286
|
+
async function withTimeout4(source, run, timeoutMs, external) {
|
|
145287
|
+
const cancelled = () => ({ ok: false, reason: "classification cancelled" });
|
|
145288
|
+
if (external?.aborted)
|
|
145289
|
+
return cancelled();
|
|
145290
|
+
if (timeoutMs <= 0)
|
|
145291
|
+
return { ok: false, reason: "classification deadline exceeded" };
|
|
144243
145292
|
const controller = new AbortController;
|
|
145293
|
+
const onExternalAbort = () => controller.abort();
|
|
145294
|
+
external?.addEventListener("abort", onExternalAbort, { once: true });
|
|
144244
145295
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
145296
|
+
let onAbort = () => {};
|
|
144245
145297
|
try {
|
|
144246
|
-
|
|
144247
|
-
|
|
144248
|
-
|
|
144249
|
-
|
|
144250
|
-
|
|
144251
|
-
]);
|
|
145298
|
+
const aborted = new Promise((resolve) => {
|
|
145299
|
+
onAbort = () => resolve(external?.aborted ? cancelled() : { ok: false, reason: `${source} classifier timed out after ${timeoutMs}ms` });
|
|
145300
|
+
controller.signal.addEventListener("abort", onAbort, { once: true });
|
|
145301
|
+
});
|
|
145302
|
+
return await Promise.race([run(controller.signal), aborted]);
|
|
144252
145303
|
} catch (error) {
|
|
144253
145304
|
return { ok: false, reason: `${source} classifier errored: ${error instanceof Error ? error.message : String(error)}` };
|
|
144254
145305
|
} finally {
|
|
144255
145306
|
clearTimeout(timer);
|
|
145307
|
+
external?.removeEventListener("abort", onExternalAbort);
|
|
145308
|
+
controller.signal.removeEventListener("abort", onAbort);
|
|
145309
|
+
controller.abort();
|
|
144256
145310
|
}
|
|
144257
145311
|
}
|
|
144258
145312
|
async function classifyTurn(line, categories, opts = {}) {
|
|
144259
145313
|
if (isSlashCommandLine(line) || categories.length === 0)
|
|
144260
145314
|
return;
|
|
145315
|
+
if (opts.signal?.aborted)
|
|
145316
|
+
return { result: { ok: false, reason: "classification cancelled" }, trace: [] };
|
|
144261
145317
|
const deterministic = classifyDeterministic(line);
|
|
144262
145318
|
if (deterministic !== undefined && categories.includes(deterministic)) {
|
|
144263
145319
|
const result = { ok: true, category: deterministic, confidence: 1, source: "deterministic" };
|
|
144264
145320
|
return { result, trace: [{ source: "deterministic", result }] };
|
|
144265
145321
|
}
|
|
144266
145322
|
const timeoutMs = opts.timeoutMs ?? CLASSIFY_TURN_TIMEOUT_MS;
|
|
145323
|
+
const deadline = performance.now() + Math.max(0, timeoutMs);
|
|
145324
|
+
const remaining = () => Math.max(0, deadline - performance.now());
|
|
145325
|
+
const fallback = opts.fallbackModel !== undefined ? opts.fallbackModel : opts.sessionProvider !== undefined && opts.sessionModel !== undefined ? { providerId: opts.sessionProvider, modelId: opts.sessionModel } : undefined;
|
|
144267
145326
|
const trace = [];
|
|
145327
|
+
const stopped = () => opts.signal?.aborted ? { result: { ok: false, reason: "classification cancelled" }, trace } : remaining() <= 0 ? { result: { ok: false, reason: "classification deadline exceeded" }, trace } : undefined;
|
|
145328
|
+
const initialStop = stopped();
|
|
145329
|
+
if (initialStop)
|
|
145330
|
+
return initialStop;
|
|
144268
145331
|
if (opts.jevEnabled === true) {
|
|
144269
145332
|
const credential = resolveJevClassifierCredential(opts.env, opts.dir);
|
|
144270
145333
|
if (credential.key !== undefined) {
|
|
144271
145334
|
const jev = new JevTaskClassifier({ ...opts.fetch !== undefined ? { fetch: opts.fetch } : {}, ...opts.env !== undefined ? { env: opts.env } : {}, ...opts.dir !== undefined ? { dir: opts.dir } : {}, timeoutMs });
|
|
144272
|
-
const jevResult = await withTimeout4("jev", (signal) => jev.classify(line, categories, { signal }), timeoutMs);
|
|
145335
|
+
const jevResult = await withTimeout4("jev", (signal) => jev.classify(line, categories, { signal }), Math.min(remaining(), fallback != null ? timeoutMs / 2 : timeoutMs), opts.signal);
|
|
144273
145336
|
trace.push({ source: "jev", result: jevResult });
|
|
145337
|
+
const jevStop = stopped();
|
|
145338
|
+
if (jevStop)
|
|
145339
|
+
return jevStop;
|
|
144274
145340
|
if (jevResult.ok)
|
|
144275
145341
|
return { result: jevResult, trace };
|
|
144276
145342
|
}
|
|
144277
145343
|
}
|
|
144278
|
-
if (
|
|
145344
|
+
if (fallback != null) {
|
|
144279
145345
|
const mainModel = new MainModelTaskClassifier({
|
|
144280
|
-
provider:
|
|
144281
|
-
model:
|
|
145346
|
+
provider: fallback.providerId,
|
|
145347
|
+
model: fallback.modelId,
|
|
144282
145348
|
...opts.env !== undefined ? { env: opts.env } : {},
|
|
144283
145349
|
...opts.fetch !== undefined ? { fetch: opts.fetch } : {},
|
|
144284
145350
|
...opts.providerFactory !== undefined ? { providerFactory: opts.providerFactory } : {}
|
|
144285
145351
|
});
|
|
144286
|
-
const mainModelResult = await withTimeout4("main-model", (signal) => mainModel.classify(line, categories, { signal }),
|
|
145352
|
+
const mainModelResult = await withTimeout4("main-model", (signal) => mainModel.classify(line, categories, { signal }), remaining(), opts.signal);
|
|
144287
145353
|
trace.push({ source: "main-model", result: mainModelResult });
|
|
145354
|
+
const mainStop = stopped();
|
|
145355
|
+
if (mainStop)
|
|
145356
|
+
return mainStop;
|
|
144288
145357
|
if (mainModelResult.ok)
|
|
144289
145358
|
return { result: mainModelResult, trace };
|
|
144290
145359
|
}
|
|
@@ -144298,42 +145367,123 @@ var init_classify_turn = __esm(() => {
|
|
|
144298
145367
|
init_main_model_classifier();
|
|
144299
145368
|
});
|
|
144300
145369
|
|
|
145370
|
+
// src/harness/routing/classifier-model.ts
|
|
145371
|
+
function selectClassifierModel(detected, profiles, allowed) {
|
|
145372
|
+
const candidates = [];
|
|
145373
|
+
for (const provider of detected) {
|
|
145374
|
+
for (const modelId of provider.models ?? []) {
|
|
145375
|
+
const profile = profiles[profileKey(provider.name, modelId)];
|
|
145376
|
+
if (!profile?.available || !isModelDerivable(profile))
|
|
145377
|
+
continue;
|
|
145378
|
+
if (profile.strengthTier.value === "light" || profile.strengthTier.source === "unknown" || profile.strengthTier.source === "guessed")
|
|
145379
|
+
continue;
|
|
145380
|
+
if (allowed(provider.name, modelId))
|
|
145381
|
+
candidates.push(profile);
|
|
145382
|
+
}
|
|
145383
|
+
}
|
|
145384
|
+
candidates.sort((a, b) => Number(a.strengthTier.value === "deep") - Number(b.strengthTier.value === "deep") || b.priority.value - a.priority.value || profileKey(a.providerId, a.modelId).localeCompare(profileKey(b.providerId, b.modelId)));
|
|
145385
|
+
const selected = candidates[0];
|
|
145386
|
+
return selected === undefined ? undefined : { providerId: selected.providerId, modelId: selected.modelId };
|
|
145387
|
+
}
|
|
145388
|
+
var init_classifier_model = __esm(() => {
|
|
145389
|
+
init_model_profile();
|
|
145390
|
+
});
|
|
145391
|
+
|
|
144301
145392
|
// src/tui/routing-classifier-source.ts
|
|
144302
145393
|
async function runRoutingClassifierForTurn(line, opts) {
|
|
145394
|
+
if (opts.signal?.aborted)
|
|
145395
|
+
return;
|
|
144303
145396
|
if (!opts.enabled)
|
|
144304
145397
|
return;
|
|
145398
|
+
const profiles = loadModelProfiles(opts.userConfigDir);
|
|
145399
|
+
const env = envWithSavedApiKeys(opts.env ?? process.env);
|
|
145400
|
+
const external = await resolveExternalSetting({ cwd: opts.cwd, ...opts.userConfigDir !== undefined ? { dir: opts.userConfigDir } : {} });
|
|
145401
|
+
if (opts.signal?.aborted)
|
|
145402
|
+
return;
|
|
145403
|
+
const externalAllowed = externalAllowedConnectedPredicate(connectedPredicateFrom(opts.detected), external.value === "on", loadExternalProvidersConfig(opts.userConfigDir).config);
|
|
145404
|
+
const fallbackModel = selectClassifierModel(opts.detected, profiles, (providerId, modelId) => externalAllowed(providerId, modelId) && (opts.classifierAllowed?.(providerId, modelId) ?? true) && (opts.providerFactory !== undefined || hasCredential(providerId, env)));
|
|
144305
145405
|
const classification = await classifyTurn(line, TURN_CLASSIFIER_CATEGORIES, {
|
|
144306
|
-
jevEnabled: opts.jevEnabled,
|
|
144307
|
-
|
|
144308
|
-
|
|
145406
|
+
jevEnabled: opts.jevEnabled && opts.jevAllowed !== false,
|
|
145407
|
+
...opts.signal !== undefined ? { signal: opts.signal } : {},
|
|
145408
|
+
fallbackModel: fallbackModel ?? null,
|
|
145409
|
+
...opts.providerFactory !== undefined ? { providerFactory: opts.providerFactory } : {},
|
|
144309
145410
|
...opts.env !== undefined ? { env: opts.env } : {},
|
|
144310
145411
|
...opts.fetch !== undefined ? { fetch: opts.fetch } : {},
|
|
144311
145412
|
timeoutMs: opts.timeoutMs ?? CLASSIFY_TURN_TIMEOUT_MS
|
|
144312
145413
|
});
|
|
144313
|
-
if (
|
|
145414
|
+
if (opts.signal?.aborted)
|
|
145415
|
+
return;
|
|
145416
|
+
if (classification === undefined)
|
|
144314
145417
|
return;
|
|
145418
|
+
const baseline = { providerId: opts.sessionProvider, modelId: opts.sessionModel };
|
|
145419
|
+
const skippedJev = opts.jevEnabled && opts.jevAllowed === false ? "jev: caller policy denied" : opts.jevEnabled && resolveJevClassifierCredential(opts.env).key === undefined ? "jev: credentials unavailable" : undefined;
|
|
145420
|
+
if (!classification.result.ok) {
|
|
145421
|
+
const failures = classification.trace.flatMap((stage) => stage.result.ok ? [] : [`${stage.source}: ${stage.result.reason}`]);
|
|
145422
|
+
return {
|
|
145423
|
+
classification,
|
|
145424
|
+
baseline,
|
|
145425
|
+
assignment: { kind: "session-default" },
|
|
145426
|
+
fallbackReason: redactSensitiveText([
|
|
145427
|
+
skippedJev,
|
|
145428
|
+
...failures,
|
|
145429
|
+
fallbackModel === undefined ? "no available authorized sufficient classifier" : undefined,
|
|
145430
|
+
classification.result.reason
|
|
145431
|
+
].filter(Boolean).join("; "))
|
|
145432
|
+
};
|
|
145433
|
+
}
|
|
144315
145434
|
const category = classification.result.category;
|
|
144316
145435
|
const location = { cwd: opts.cwd, ...opts.userConfigDir !== undefined ? { userConfigDir: opts.userConfigDir } : {} };
|
|
144317
145436
|
const [project, user] = await Promise.all([loadRoutingConfig("project", location), loadRoutingConfig("user", location)]);
|
|
145437
|
+
if (opts.signal?.aborted)
|
|
145438
|
+
return;
|
|
144318
145439
|
const connected = connectedPredicateFrom(opts.detected);
|
|
144319
|
-
const profiles = loadModelProfiles(opts.userConfigDir);
|
|
144320
145440
|
const available = availablePredicateFromProfiles(profiles);
|
|
145441
|
+
let fallbackReason;
|
|
145442
|
+
const connectedForTurn = (providerId, modelId) => {
|
|
145443
|
+
if (modelId !== undefined) {
|
|
145444
|
+
const reason = !connected(providerId, modelId) ? "model not connected" : !available(providerId, modelId) ? "model unavailable" : !(opts.executorAllowed?.(providerId, modelId) ?? true) ? "executor policy denied" : undefined;
|
|
145445
|
+
if (reason !== undefined) {
|
|
145446
|
+
fallbackReason ??= `${providerId}/${modelId}: ${reason}`;
|
|
145447
|
+
return false;
|
|
145448
|
+
}
|
|
145449
|
+
return true;
|
|
145450
|
+
}
|
|
145451
|
+
const defaultId = resolveProviderDefaultModelId(providerId);
|
|
145452
|
+
const reason = !connected(providerId) ? "provider not connected" : defaultId === undefined ? "no documented provider default" : !connected(providerId, defaultId) ? "default model not connected" : !available(providerId, defaultId) ? "default model unavailable" : !(opts.executorAllowed?.(providerId, defaultId) ?? true) ? "executor policy denied" : opts.providerFactory === undefined && !hasCredential(providerId, env) ? "credentials unavailable" : undefined;
|
|
145453
|
+
if (reason !== undefined) {
|
|
145454
|
+
fallbackReason ??= `${providerId} (provider default): ${reason}`;
|
|
145455
|
+
return false;
|
|
145456
|
+
}
|
|
145457
|
+
return true;
|
|
145458
|
+
};
|
|
144321
145459
|
const sessionProvider = opts.detected.find((p) => p.name === opts.sessionProvider);
|
|
144322
145460
|
const models = sessionProvider?.models ?? [opts.sessionModel];
|
|
144323
145461
|
const derived = deriveDefaultTable(opts.sessionProvider, models, profiles, opts.sessionModel);
|
|
144324
|
-
const resolved = resolveCategoryDetailed(category, { project: project.table, user: user.table, derived },
|
|
144325
|
-
|
|
144326
|
-
|
|
144327
|
-
|
|
145462
|
+
const resolved = resolveCategoryDetailed(category, { project: project.table, user: user.table, derived }, connectedForTurn, available);
|
|
145463
|
+
fallbackReason ??= resolved.rejected === undefined ? skippedJev : describeRejectionNotice(resolved.rejected, resolved.assignment);
|
|
145464
|
+
const stageFailures = classification.trace.flatMap((stage) => stage.result.ok ? [] : [`${stage.source}: ${stage.result.reason}`]);
|
|
145465
|
+
const reasons = [...new Set([skippedJev, ...stageFailures, fallbackReason].filter((reason) => reason !== undefined))];
|
|
145466
|
+
fallbackReason = reasons.length === 0 ? undefined : redactSensitiveText(reasons.join("; "));
|
|
145467
|
+
const result = {
|
|
145468
|
+
classification,
|
|
145469
|
+
category,
|
|
145470
|
+
baseline,
|
|
145471
|
+
assignment: resolved.assignment,
|
|
145472
|
+
...fallbackReason !== undefined ? { fallbackReason: redactSensitiveText(fallbackReason) } : {}
|
|
145473
|
+
};
|
|
145474
|
+
if (resolved.assignment.kind === "session-default")
|
|
145475
|
+
return result;
|
|
144328
145476
|
if (resolved.assignment.kind === "model") {
|
|
144329
|
-
return {
|
|
145477
|
+
return { ...result, routed: { providerId: resolved.assignment.providerId, modelId: resolved.assignment.modelId } };
|
|
144330
145478
|
}
|
|
144331
|
-
|
|
145479
|
+
const modelId = resolveProviderDefaultModelId(resolved.assignment.providerId);
|
|
145480
|
+
return modelId === undefined ? result : { ...result, routed: { providerId: resolved.assignment.providerId, modelId } };
|
|
144332
145481
|
}
|
|
144333
145482
|
function renderRoutingFallbackLine(result) {
|
|
144334
145483
|
if (result === undefined)
|
|
144335
145484
|
return "[route fallback: no classifier category; using session model]";
|
|
144336
|
-
|
|
145485
|
+
const target = result.baseline === undefined ? "session model" : `${result.baseline.providerId}/${result.baseline.modelId}`;
|
|
145486
|
+
return redactSensitiveText(`[route fallback: ${result.category ?? "no classifier category"} -> ${describeAssignment(result.assignment)}${result.fallbackReason !== undefined ? `; ${result.fallbackReason}` : ""}; using ${target}]`);
|
|
144337
145487
|
}
|
|
144338
145488
|
function pct6(n) {
|
|
144339
145489
|
return `${Math.round(n * 100)}%`;
|
|
@@ -144343,7 +145493,7 @@ function renderRoutingTagLine(result) {
|
|
|
144343
145493
|
return;
|
|
144344
145494
|
const source = result.classification.result.ok ? result.classification.result.source : "none";
|
|
144345
145495
|
const confidence = result.classification.result.ok ? ` ${pct6(result.classification.result.confidence)}` : "";
|
|
144346
|
-
return `[${result.category} -> ${result.routed.providerId}/${result.routed.modelId}] (${source}${confidence})`;
|
|
145496
|
+
return `[${result.category} -> ${result.routed.providerId}/${result.routed.modelId}] (${source}${confidence}${result.fallbackReason !== undefined ? `; fallback: ${result.fallbackReason}` : ""})`;
|
|
144347
145497
|
}
|
|
144348
145498
|
function renderRoutingSidebarValue(enabled, routedCount, lastCategory) {
|
|
144349
145499
|
if (!enabled)
|
|
@@ -144356,6 +145506,14 @@ var init_routing_classifier_source = __esm(() => {
|
|
|
144356
145506
|
init_table();
|
|
144357
145507
|
init_config2();
|
|
144358
145508
|
init_derive_default_table();
|
|
145509
|
+
init_provider_default();
|
|
145510
|
+
init_classifier_model();
|
|
145511
|
+
init_single_turn();
|
|
145512
|
+
init_shell_config();
|
|
145513
|
+
init_service2();
|
|
145514
|
+
init_jev_classifier();
|
|
145515
|
+
init_external_switch();
|
|
145516
|
+
init_external_providers();
|
|
144359
145517
|
init_model_profile();
|
|
144360
145518
|
TURN_CLASSIFIER_CATEGORIES = ROUTING_CATEGORIES.filter((c) => c !== "default");
|
|
144361
145519
|
});
|
|
@@ -155445,6 +156603,15 @@ var init_digest_summary = __esm(() => {
|
|
|
155445
156603
|
import { randomUUID as randomUUID41 } from "crypto";
|
|
155446
156604
|
import { mkdir as mkdir68, mkdtemp as mkdtemp10, rm as rm26, writeFile as writeFile44 } from "fs/promises";
|
|
155447
156605
|
import path303 from "path";
|
|
156606
|
+
function scrubDigestReportLocator(locator, env) {
|
|
156607
|
+
const scrubbed = scrubGrantedOutput(locator, env);
|
|
156608
|
+
const basename = path303.basename(locator);
|
|
156609
|
+
if (!/^dig-\d{8}T\d{6}-[a-f0-9]{8}\.md$/.test(basename))
|
|
156610
|
+
return scrubbed;
|
|
156611
|
+
const from = locator.length - basename.length;
|
|
156612
|
+
const phones = detectPii(locator).filter((m) => m.mask === "phone" && m.start >= from);
|
|
156613
|
+
return scrubbed === applyRedaction(locator, phones) ? locator : scrubbed;
|
|
156614
|
+
}
|
|
155448
156615
|
function capForDelivery(text, reportPath) {
|
|
155449
156616
|
if (text.length <= DELIVERY_TEXT_MAX)
|
|
155450
156617
|
return text;
|
|
@@ -155618,8 +156785,8 @@ async function runLocked2(projectRoot, entry, action, deps, runId, now) {
|
|
|
155618
156785
|
const statusReason = stopReason ?? (crashed !== undefined ? `the run threw: ${crashed}` : everySourceFailed ? "no source could be read" : undefined);
|
|
155619
156786
|
const statusDetail = everySourceFailed && stopReason === undefined && crashed === undefined ? partial.slice(0, 6).map((f) => `
|
|
155620
156787
|
- ${f.source}: ${f.detail}`).join("") : "";
|
|
155621
|
-
const message = statusReason !== undefined ? scrubGrantedOutput(`Digest ${entry.name} stopped \u2014 ${statusReason}${statusDetail}${reportPath !== undefined ? `
|
|
155622
|
-
Report: ${reportPath}` : ""}
|
|
156788
|
+
const message = statusReason !== undefined ? `${scrubGrantedOutput(`Digest ${entry.name} stopped \u2014 ${statusReason}${statusDetail}`, env)}${reportPath !== undefined ? `
|
|
156789
|
+
Report: ${scrubDigestReportLocator(reportPath, env)}` : ""}` : capForDelivery(scrubGrantedOutput(summaryText !== undefined ? `Summary (written by the model)
|
|
155623
156790
|
${summaryText}
|
|
155624
156791
|
|
|
155625
156792
|
${body}` : body, env), reportPath);
|
|
@@ -155696,6 +156863,7 @@ var init_digest_run = __esm(() => {
|
|
|
155696
156863
|
init_trigger_dispatch();
|
|
155697
156864
|
init_fs();
|
|
155698
156865
|
init_maintenance_lock();
|
|
156866
|
+
init_service2();
|
|
155699
156867
|
init_digest_config();
|
|
155700
156868
|
init_granted_tools();
|
|
155701
156869
|
init_schedule_verify();
|
|
@@ -159986,7 +161154,6 @@ async function launchTuiAgentShell(opts) {
|
|
|
159986
161154
|
let routingEnabled = loadShellConfig().routingClassifier?.enabled === true;
|
|
159987
161155
|
let routingRoutedCount = 0;
|
|
159988
161156
|
let routingLastCategory;
|
|
159989
|
-
let sessionModelExplicit = false;
|
|
159990
161157
|
const refreshRoutingSidebar = () => {
|
|
159991
161158
|
clearTranscriptChildren(sbRoute);
|
|
159992
161159
|
if (!routingEnabled)
|
|
@@ -161877,7 +163044,6 @@ ${formatThemeList(getThemeId())}`);
|
|
|
161877
163044
|
};
|
|
161878
163045
|
const switchTo = async (ns) => {
|
|
161879
163046
|
currentSel = ns;
|
|
161880
|
-
sessionModelExplicit = true;
|
|
161881
163047
|
deps = {
|
|
161882
163048
|
...await opts.makeAgentDeps(ns, liveSlateSession, busClientRef),
|
|
161883
163049
|
onContextCompaction,
|
|
@@ -163365,15 +164531,13 @@ ${formatThemeList(getThemeId())}`);
|
|
|
163365
164531
|
const sessionDeps = deps;
|
|
163366
164532
|
(async () => {
|
|
163367
164533
|
let deps = sessionDeps;
|
|
164534
|
+
const turnSignal = foregroundOperation.signal;
|
|
163368
164535
|
let routingOutcome;
|
|
163369
|
-
if (routingEnabled &&
|
|
163370
|
-
io.onSystem?.(`[route skipped: manual /model selection pins this session]
|
|
163371
|
-
`);
|
|
163372
|
-
}
|
|
163373
|
-
if (routingEnabled && !sessionModelExplicit && origin === "operator" && line.trim().length > 0) {
|
|
164536
|
+
if (routingEnabled && origin === "operator" && line.trim().length > 0) {
|
|
163374
164537
|
try {
|
|
163375
164538
|
routingOutcome = await runRoutingClassifierForTurn(line, {
|
|
163376
164539
|
enabled: routingEnabled,
|
|
164540
|
+
signal: turnSignal,
|
|
163377
164541
|
jevEnabled: routingEnabled,
|
|
163378
164542
|
cwd: opts.session?.cwd ?? process.cwd(),
|
|
163379
164543
|
detected: opts.detected ?? [],
|
|
@@ -163381,7 +164545,7 @@ ${formatThemeList(getThemeId())}`);
|
|
|
163381
164545
|
sessionModel: currentSel.model
|
|
163382
164546
|
});
|
|
163383
164547
|
} catch {}
|
|
163384
|
-
if (routingOutcome?.routed !== undefined) {
|
|
164548
|
+
if (!turnSignal.aborted && routingOutcome?.routed !== undefined) {
|
|
163385
164549
|
try {
|
|
163386
164550
|
deps = {
|
|
163387
164551
|
...await opts.makeAgentDeps({ provider: routingOutcome.routed.providerId, model: routingOutcome.routed.modelId }, liveSlateSession, busClientRef),
|
|
@@ -163390,19 +164554,22 @@ ${formatThemeList(getThemeId())}`);
|
|
|
163390
164554
|
...sessionDeps.busAck !== undefined ? { busAck: sessionDeps.busAck } : {},
|
|
163391
164555
|
...sessionDeps.busLeases !== undefined ? { busLeases: sessionDeps.busLeases } : {}
|
|
163392
164556
|
};
|
|
163393
|
-
|
|
163394
|
-
|
|
163395
|
-
|
|
163396
|
-
|
|
163397
|
-
|
|
163398
|
-
|
|
164557
|
+
if (!turnSignal.aborted) {
|
|
164558
|
+
routingRoutedCount += 1;
|
|
164559
|
+
routingLastCategory = routingOutcome.category;
|
|
164560
|
+
refreshRoutingSidebar();
|
|
164561
|
+
const tag = renderRoutingTagLine(routingOutcome);
|
|
164562
|
+
if (tag !== undefined)
|
|
164563
|
+
io.onSystem?.(`${tag}
|
|
163399
164564
|
`);
|
|
164565
|
+
}
|
|
163400
164566
|
} catch {
|
|
163401
164567
|
deps = sessionDeps;
|
|
163402
|
-
|
|
164568
|
+
if (!turnSignal.aborted)
|
|
164569
|
+
io.onSystem?.(`[route fallback: selected model could not be prepared; check availability and external policy; using session model]
|
|
163403
164570
|
`);
|
|
163404
164571
|
}
|
|
163405
|
-
} else {
|
|
164572
|
+
} else if (!turnSignal.aborted) {
|
|
163406
164573
|
io.onSystem?.(`${renderRoutingFallbackLine(routingOutcome)}
|
|
163407
164574
|
`);
|
|
163408
164575
|
}
|
|
@@ -163412,12 +164579,11 @@ ${formatThemeList(getThemeId())}`);
|
|
|
163412
164579
|
syncArchive();
|
|
163413
164580
|
rewindRecorder.beginTurn({ archiveIndex: archive.length, prompt: line });
|
|
163414
164581
|
const foregroundIo = createForegroundAgentIoFacade(foregroundOperation, operation, io, telegramTurn ? telegramTurnIo : {});
|
|
163415
|
-
|
|
163416
|
-
runAgentTurn(foregroundIo, deps, history, line, {
|
|
164582
|
+
(turnSignal.aborted ? Promise.resolve() : runAgentTurn(foregroundIo, deps, history, line, {
|
|
163417
164583
|
signal: turnSignal,
|
|
163418
164584
|
...origin === "operator" ? {} : { origin },
|
|
163419
164585
|
...slateSession !== undefined ? { slateSession, pruneArchive: true } : {}
|
|
163420
|
-
}).finally(() => {
|
|
164586
|
+
})).finally(() => {
|
|
163421
164587
|
foregroundOperation.settle(operation);
|
|
163422
164588
|
remoteQueue.turnSettled({ failed: turnFailed, aborted: turnSignal.aborted });
|
|
163423
164589
|
if (foregroundOperation.isDisposed)
|
|
@@ -185268,10 +186434,10 @@ function codeOf(text) {
|
|
|
185268
186434
|
return stripTrailingComment2(text).trimEnd();
|
|
185269
186435
|
}
|
|
185270
186436
|
function numericSkeleton(text) {
|
|
185271
|
-
return text.replace(
|
|
186437
|
+
return text.replace(NUMBER3, "\x00");
|
|
185272
186438
|
}
|
|
185273
186439
|
function numbersOf(text) {
|
|
185274
|
-
return [...text.matchAll(
|
|
186440
|
+
return [...text.matchAll(NUMBER3)].map((match) => ({ value: Number(match[0]), index: match.index ?? 0 }));
|
|
185275
186441
|
}
|
|
185276
186442
|
function enclosingNames(code, index) {
|
|
185277
186443
|
const names = [];
|
|
@@ -185534,7 +186700,7 @@ function renderFloorMarkdown(report) {
|
|
|
185534
186700
|
return lines.join(`
|
|
185535
186701
|
`);
|
|
185536
186702
|
}
|
|
185537
|
-
var FLOOR_FINDING_KINDS, COMMENT_OPENER, FLOOR_WORDS, CEILING_WORDS, CAPACITY_WORDS,
|
|
186703
|
+
var FLOOR_FINDING_KINDS, COMMENT_OPENER, FLOOR_WORDS, CEILING_WORDS, CAPACITY_WORDS, NUMBER3, KEY_SEPARATORS, TRAILING_IDENTIFIER, TRAILING_INDEX, OPENS_CONTAINER, TYPE_KEYWORDS, COMMENTED_LINE, MODIFIER_SEGMENT, DISABLED_CALL, DISABLED_ANNOTATION, OTHER_TEST_PATHS, ASSERTION_HEAD, SUPPRESSION_MARKERS, ANNOTATION_MARKERS;
|
|
185538
186704
|
var init_floor = __esm(() => {
|
|
185539
186705
|
init_selection();
|
|
185540
186706
|
FLOOR_FINDING_KINDS = [
|
|
@@ -185624,7 +186790,7 @@ var init_floor = __esm(() => {
|
|
|
185624
186790
|
"parallel",
|
|
185625
186791
|
"batch"
|
|
185626
186792
|
]);
|
|
185627
|
-
|
|
186793
|
+
NUMBER3 = /[0-9]+(?:\.[0-9]+)?/g;
|
|
185628
186794
|
KEY_SEPARATORS = /[\s:=(["'`{<]*$/;
|
|
185629
186795
|
TRAILING_IDENTIFIER = /[A-Za-z0-9_$.-]+$/;
|
|
185630
186796
|
TRAILING_INDEX = /\[\s*[0-9]*\s*\]$/;
|
|
@@ -201605,7 +202771,22 @@ async function startServeListener(input) {
|
|
|
201605
202771
|
} catch (error) {
|
|
201606
202772
|
console.error(`keryx serve: remote surface did not close cleanly: ${error instanceof Error ? error.message : String(error)}`);
|
|
201607
202773
|
}
|
|
201608
|
-
|
|
202774
|
+
if (input.remote === undefined) {
|
|
202775
|
+
await server.stop(true);
|
|
202776
|
+
} else {
|
|
202777
|
+
let timer;
|
|
202778
|
+
try {
|
|
202779
|
+
await Promise.race([
|
|
202780
|
+
server.stop(false),
|
|
202781
|
+
new Promise((resolve) => {
|
|
202782
|
+
timer = setTimeout(resolve, 250);
|
|
202783
|
+
})
|
|
202784
|
+
]);
|
|
202785
|
+
await server.stop(true);
|
|
202786
|
+
} finally {
|
|
202787
|
+
clearTimeout(timer);
|
|
202788
|
+
}
|
|
202789
|
+
}
|
|
201609
202790
|
state = "stopped";
|
|
201610
202791
|
}
|
|
201611
202792
|
}
|