@oh-my-pi/pi-coding-agent 14.3.0 → 14.4.1

package/src/edit/line-hash.ts

@@ -3,26 +3,740 @@
  * circular dependencies (prompt-templates → hashline → tools → edit).
  */
 
-/** 16-char nibble alphabet (no digits); shared with chunk checksum suffixes. */
-export const HASHLINE_NIBBLE_ALPHABET = "ZPMQVRWSNKTXJBYH";
+/**
+ * 647 single-token BPE bigrams for hashline anchors. Every entry tokenizes as
+ * exactly one token in modern BPE vocabularies (cl100k / o200k / Claude family),
+ * so a hashline anchor built from one bigram is exactly 1 token.
+ *
+ * This is the complete set of 2-letter lowercase combinations that are single
+ * tokens — the 29 missing combinations are rare-letter pairs (q/x/z heavy)
+ * that no major BPE vocabulary merges into a single token.
+ *
+ * Order is stable forever — changing it would invalidate every saved
+ * `LINE+ID` reference in transcripts and prompts.
+ */
+export const HASHLINE_BIGRAMS = [
+	"aa",
+	"ab",
+	"ac",
+	"ad",
+	"ae",
+	"af",
+	"ag",
+	"ah",
+	"ai",
+	"aj",
+	"ak",
+	"al",
+	"am",
+	"an",
+	"ao",
+	"ap",
+	"aq",
+	"ar",
+	"as",
+	"at",
+	"au",
+	"av",
+	"aw",
+	"ax",
+	"ay",
+	"az",
+	"ba",
+	"bb",
+	"bc",
+	"bd",
+	"be",
+	"bf",
+	"bg",
+	"bh",
+	"bi",
+	"bj",
+	"bk",
+	"bl",
+	"bm",
+	"bn",
+	"bo",
+	"bp",
+	"br",
+	"bs",
+	"bt",
+	"bu",
+	"bv",
+	"bw",
+	"bx",
+	"by",
+	"bz",
+	"ca",
+	"cb",
+	"cc",
+	"cd",
+	"ce",
+	"cf",
+	"cg",
+	"ch",
+	"ci",
+	"cj",
+	"ck",
+	"cl",
+	"cm",
+	"cn",
+	"co",
+	"cp",
+	"cq",
+	"cr",
+	"cs",
+	"ct",
+	"cu",
+	"cv",
+	"cw",
+	"cx",
+	"cy",
+	"cz",
+	"da",
+	"db",
+	"dc",
+	"dd",
+	"de",
+	"df",
+	"dg",
+	"dh",
+	"di",
+	"dj",
+	"dk",
+	"dl",
+	"dm",
+	"dn",
+	"do",
+	"dp",
+	"dq",
+	"dr",
+	"ds",
+	"dt",
+	"du",
+	"dv",
+	"dw",
+	"dx",
+	"dy",
+	"dz",
+	"ea",
+	"eb",
+	"ec",
+	"ed",
+	"ee",
+	"ef",
+	"eg",
+	"eh",
+	"ei",
+	"ej",
+	"ek",
+	"el",
+	"em",
+	"en",
+	"eo",
+	"ep",
+	"eq",
+	"er",
+	"es",
+	"et",
+	"eu",
+	"ev",
+	"ew",
+	"ex",
+	"ey",
+	"ez",
+	"fa",
+	"fb",
+	"fc",
+	"fd",
+	"fe",
+	"ff",
+	"fg",
+	"fh",
+	"fi",
+	"fj",
+	"fk",
+	"fl",
+	"fm",
+	"fn",
+	"fo",
+	"fp",
+	"fq",
+	"fr",
+	"fs",
+	"ft",
+	"fu",
+	"fv",
+	"fw",
+	"fx",
+	"fy",
+	"fz",
+	"ga",
+	"gb",
+	"gc",
+	"gd",
+	"ge",
+	"gf",
+	"gg",
+	"gh",
+	"gi",
+	"gj",
+	"gl",
+	"gm",
+	"gn",
+	"go",
+	"gp",
+	"gr",
+	"gs",
+	"gt",
+	"gu",
+	"gv",
+	"gw",
+	"gx",
+	"gy",
+	"gz",
+	"ha",
+	"hb",
+	"hc",
+	"hd",
+	"he",
+	"hf",
+	"hg",
+	"hh",
+	"hi",
+	"hj",
+	"hk",
+	"hl",
+	"hm",
+	"hn",
+	"ho",
+	"hp",
+	"hq",
+	"hr",
+	"hs",
+	"ht",
+	"hu",
+	"hv",
+	"hw",
+	"hx",
+	"hy",
+	"hz",
+	"ia",
+	"ib",
+	"ic",
+	"id",
+	"ie",
+	"if",
+	"ig",
+	"ih",
+	"ii",
+	"ij",
+	"ik",
+	"il",
+	"im",
+	"in",
+	"io",
+	"ip",
+	"iq",
+	"ir",
+	"is",
+	"it",
+	"iu",
+	"iv",
+	"iw",
+	"ix",
+	"iy",
+	"iz",
+	"ja",
+	"jb",
+	"jc",
+	"jd",
+	"je",
+	"jf",
+	"jg",
+	"jh",
+	"ji",
+	"jj",
+	"jk",
+	"jl",
+	"jm",
+	"jn",
+	"jo",
+	"jp",
+	"jq",
+	"jr",
+	"js",
+	"jt",
+	"ju",
+	"jw",
+	"jx",
+	"jy",
+	"ka",
+	"kb",
+	"kc",
+	"kd",
+	"ke",
+	"kf",
+	"kg",
+	"kh",
+	"ki",
+	"kj",
+	"kk",
+	"kl",
+	"km",
+	"kn",
+	"ko",
+	"kp",
+	"kr",
+	"ks",
+	"kt",
+	"ku",
+	"kv",
+	"kw",
+	"kx",
+	"ky",
+	"la",
+	"lb",
+	"lc",
+	"ld",
+	"le",
+	"lf",
+	"lg",
+	"lh",
+	"li",
+	"lj",
+	"lk",
+	"ll",
+	"lm",
+	"ln",
+	"lo",
+	"lp",
+	"lr",
+	"ls",
+	"lt",
+	"lu",
+	"lv",
+	"lw",
+	"lx",
+	"ly",
+	"lz",
+	"ma",
+	"mb",
+	"mc",
+	"md",
+	"me",
+	"mf",
+	"mg",
+	"mh",
+	"mi",
+	"mj",
+	"mk",
+	"ml",
+	"mm",
+	"mn",
+	"mo",
+	"mp",
+	"mq",
+	"mr",
+	"ms",
+	"mt",
+	"mu",
+	"mv",
+	"mw",
+	"mx",
+	"my",
+	"mz",
+	"na",
+	"nb",
+	"nc",
+	"nd",
+	"ne",
+	"nf",
+	"ng",
+	"nh",
+	"ni",
+	"nj",
+	"nk",
+	"nl",
+	"nm",
+	"nn",
+	"no",
+	"np",
+	"nr",
+	"ns",
+	"nt",
+	"nu",
+	"nv",
+	"nw",
+	"nx",
+	"ny",
+	"nz",
+	"oa",
+	"ob",
+	"oc",
+	"od",
+	"oe",
+	"of",
+	"og",
+	"oh",
+	"oi",
+	"oj",
+	"ok",
+	"ol",
+	"om",
+	"on",
+	"oo",
+	"op",
+	"oq",
+	"or",
+	"os",
+	"ot",
+	"ou",
+	"ov",
+	"ow",
+	"ox",
+	"oy",
+	"oz",
+	"pa",
+	"pb",
+	"pc",
+	"pd",
+	"pe",
+	"pf",
+	"pg",
+	"ph",
+	"pi",
+	"pj",
+	"pk",
+	"pl",
+	"pm",
+	"pn",
+	"po",
+	"pp",
+	"pq",
+	"pr",
+	"ps",
+	"pt",
+	"pu",
+	"pv",
+	"pw",
+	"px",
+	"py",
+	"pz",
+	"qa",
+	"qb",
+	"qc",
+	"qd",
+	"qe",
+	"qh",
+	"qi",
+	"ql",
+	"qm",
+	"qn",
+	"qo",
+	"qp",
+	"qq",
+	"qr",
+	"qs",
+	"qt",
+	"qu",
+	"qw",
+	"qx",
+	"qy",
+	"ra",
+	"rb",
+	"rc",
+	"rd",
+	"re",
+	"rf",
+	"rg",
+	"rh",
+	"ri",
+	"rk",
+	"rl",
+	"rm",
+	"rn",
+	"ro",
+	"rp",
+	"rq",
+	"rr",
+	"rs",
+	"rt",
+	"ru",
+	"rv",
+	"rw",
+	"rx",
+	"ry",
+	"rz",
+	"sa",
+	"sb",
+	"sc",
+	"sd",
+	"se",
+	"sf",
+	"sg",
+	"sh",
+	"si",
+	"sj",
+	"sk",
+	"sl",
+	"sm",
+	"sn",
+	"so",
+	"sp",
+	"sq",
+	"sr",
+	"ss",
+	"st",
+	"su",
+	"sv",
+	"sw",
+	"sx",
+	"sy",
+	"sz",
+	"ta",
+	"tb",
+	"tc",
+	"td",
+	"te",
+	"tf",
+	"tg",
+	"th",
+	"ti",
+	"tj",
+	"tk",
+	"tl",
+	"tm",
+	"tn",
+	"to",
+	"tp",
+	"tr",
+	"ts",
+	"tt",
+	"tu",
+	"tv",
+	"tw",
+	"tx",
+	"ty",
+	"tz",
+	"ua",
+	"ub",
+	"uc",
+	"ud",
+	"ue",
+	"uf",
+	"ug",
+	"uh",
+	"ui",
+	"uj",
+	"uk",
+	"ul",
+	"um",
+	"un",
+	"uo",
+	"up",
+	"uq",
+	"ur",
+	"us",
+	"ut",
+	"uu",
+	"uv",
+	"uw",
+	"ux",
+	"uy",
+	"uz",
+	"va",
+	"vb",
+	"vc",
+	"vd",
+	"ve",
+	"vf",
+	"vg",
+	"vh",
+	"vi",
+	"vj",
+	"vk",
+	"vl",
+	"vm",
+	"vn",
+	"vo",
+	"vp",
+	"vq",
+	"vr",
+	"vs",
+	"vt",
+	"vu",
+	"vv",
+	"vw",
+	"vx",
+	"vy",
+	"vz",
+	"wa",
+	"wb",
+	"wc",
+	"wd",
+	"we",
+	"wf",
+	"wg",
+	"wh",
+	"wi",
+	"wj",
+	"wk",
+	"wl",
+	"wm",
+	"wn",
+	"wo",
+	"wp",
+	"wr",
+	"ws",
+	"wt",
+	"wu",
+	"wv",
+	"ww",
+	"wx",
+	"wy",
+	"xa",
+	"xb",
+	"xc",
+	"xd",
+	"xe",
+	"xf",
+	"xh",
+	"xi",
+	"xl",
+	"xm",
+	"xn",
+	"xo",
+	"xp",
+	"xr",
+	"xs",
+	"xt",
+	"xu",
+	"xx",
+	"xy",
+	"xz",
+	"ya",
+	"yb",
+	"yc",
+	"yd",
+	"ye",
+	"yf",
+	"yg",
+	"yh",
+	"yi",
+	"yj",
+	"yk",
+	"yl",
+	"ym",
+	"yn",
+	"yo",
+	"yp",
+	"yr",
+	"ys",
+	"yt",
+	"yu",
+	"yv",
+	"yw",
+	"yx",
+	"yy",
+	"yz",
+	"za",
+	"zb",
+	"zc",
+	"zd",
+	"ze",
+	"zf",
+	"zg",
+	"zh",
+	"zi",
+	"zk",
+	"zl",
+	"zm",
+	"zn",
+	"zo",
+	"zp",
+	"zr",
+	"zs",
+	"zt",
+	"zu",
+	"zw",
+	"zx",
+	"zy",
+	"zz",
+] as const;
+
+export const HASHLINE_BIGRAMS_COUNT = HASHLINE_BIGRAMS.length;
+
+/**
+ * 40 common English BPE bigrams used by chunk checksums (`path#checksum`).
+ * Kept separate from {@link HASHLINE_BIGRAMS} because the chunk checksum
+ * format is `path#bigram1bigram2` (4 chars from a 1600-code namespace) and
+ * is independent of the line-anchor format.
+ *
+ * Order is stable forever — changing it invalidates every saved chunk path.
+ */
+export const CHUNK_BIGRAMS = [
+	"th",
+	"he",
+	"in",
+	"er",
+	"an",
+	"re",
+	"on",
+	"at",
+	"en",
+	"nd",
+	"ti",
+	"es",
+	"or",
+	"te",
+	"of",
+	"ed",
+	"is",
+	"it",
+	"al",
+	"ar",
+	"st",
+	"to",
+	"nt",
+	"ng",
+	"se",
+	"ha",
+	"as",
+	"ou",
+	"io",
+	"le",
+	"ve",
+	"co",
+	"me",
+	"de",
+	"hi",
+	"ri",
+	"ro",
+	"ic",
+	"ne",
+	"ea",
+] as const;
+
+export const CHUNK_BIGRAMS_COUNT = CHUNK_BIGRAMS.length;
 
-const NIBBLE_STR = HASHLINE_NIBBLE_ALPHABET;
+/**
+ * Regex source matching exactly one bigram from {@link HASHLINE_BIGRAMS}.
+ * Used by hashline parsers — keep in sync with the alphabet array above.
+ */
+export const HASHLINE_BIGRAM_RE_SRC = `(?:${HASHLINE_BIGRAMS.join("|")})`;
 
-const DICT = Array.from({ length: 256 }, (_, i) => {
-	const h = i >>> 4;
-	const l = i & 0x0f;
-	return `${NIBBLE_STR[h]}${NIBBLE_STR[l]}`;
-});
+export const HASHLINE_CONTENT_SEPARATOR = "|";
 
 const RE_SIGNIFICANT = /[\p{L}\p{N}]/u;
 
 /**
- * Compute a short hexadecimal hash of a single line.
+ * Compute a short BPE-bigram hash of a single line.
  *
- * Uses xxHash32 on a trailing-whitespace-trimmed, CR-stripped line, truncated to 2 chars from
- * {@link NIBBLE_STR}. For lines containing no alphanumeric characters (only
- * punctuation/symbols/whitespace), the line number is mixed in to reduce hash collisions.
- * The line input should not include a trailing newline.
+ * Uses xxHash32 on a trailing-whitespace-trimmed, CR-stripped line, mapped into
+ * {@link HASHLINE_BIGRAMS} via modulo. For lines containing no alphanumeric
+ * characters (only punctuation/symbols/whitespace), the line number is mixed in
+ * to reduce hash collisions. The line input should not include a trailing newline.
  */
 export function computeLineHash(idx: number, line: string): string {
 	line = line.replace(/\r/g, "").trimEnd();
@@ -31,20 +745,30 @@ export function computeLineHash(idx: number, line: string): string {
 	if (!RE_SIGNIFICANT.test(line)) {
 		seed = idx;
 	}
-	return DICT[Bun.hash.xxHash32(line, seed) & 0xff];
+	return HASHLINE_BIGRAMS[Bun.hash.xxHash32(line, seed) % HASHLINE_BIGRAMS_COUNT];
 }
 
 /**
- * Formats a hash given the line number and text.
+ * Formats an anchor reference given a line number and its text.
+ * Returns `LINE+ID` (e.g., `42nd`) — no separator between number and bigram.
  */
 export function formatLineHash(line: number, lines: string): string {
-	return `${line}#${computeLineHash(line, lines)}`;
+	return `${line}${computeLineHash(line, lines)}`;
+}
+
+/**
+ * Formats a single line with a hashline anchor.
+ * Returns `LINE+ID|TEXT` (e.g., `42nd|function hi() {\n2er|  return;\n3in|}`)
+ */
+export function formatHashLine(lineNumber: number, line: string): string {
+	return `${lineNumber}${computeLineHash(lineNumber, line)}${HASHLINE_CONTENT_SEPARATOR}${line}`;
 }
 
 /**
  * Format file text with hashline prefixes for display.
  *
- * Each line becomes `LINENUM#HASH:TEXT` where LINENUM is 1-indexed.
+ * Each line becomes `LINE+ID|TEXT` where LINENUM is 1-indexed.
+ * No padding on line numbers; pipe separator between anchor and content.
  *
  * @param text - Raw file text string
  * @param startLine - First line number (1-indexed, defaults to 1)
@@ -53,15 +777,10 @@ export function formatLineHash(line: number, lines: string): string {
  * @example
  * ```
  * formatHashLines("function hi() {\n  return;\n}")
- * // "1#HH:function hi() {\n2#HH:  return;\n3#HH:}"
+ * // "1th|function hi() {\n2er|  return;\n3in|}"
  * ```
  */
 export function formatHashLines(text: string, startLine = 1): string {
 	const lines = text.split("\n");
-	return lines
-		.map((line, i) => {
-			const num = startLine + i;
-			return `${formatLineHash(num, line)}:${line}`;
-		})
-		.join("\n");
+	return lines.map((line, i) => formatHashLine(startLine + i, line)).join("\n");
 }