@oh-my-pi/pi-coding-agent 15.3.2 → 15.4.1

package/src/hashline/executor.ts

@@ -0,0 +1,239 @@
+import { ABORT_WARNING } from "./constants";
+import { HL_OP_CHARS, HL_OP_DELETE, HL_OP_INSERT_AFTER, HL_OP_INSERT_BEFORE, HL_OP_REPLACE } from "./hash";
+import {
+	cloneCursor,
+	type HashlineToken,
+	HashlineTokenizer,
+	isDeleteOpWithPayload,
+	type ParsedRange,
+} from "./tokenizer";
+import type { Anchor, HashlineCursor, HashlineEdit } from "./types";
+
+function validateRangeOrder(range: ParsedRange, lineNum: number): void {
+	if (range.end.line < range.start.line) {
+		throw new Error(`line ${lineNum}: range ${range.start.line}-${range.end.line} ends before it starts.`);
+	}
+}
+
+function expandRange(range: ParsedRange): Anchor[] {
+	const anchors: Anchor[] = [];
+	for (let line = range.start.line; line <= range.end.line; line++) {
+		anchors.push({ line });
+	}
+	return anchors;
+}
+
+type PendingOp =
+	| { kind: "insert"; cursor: HashlineCursor; lineNum: number }
+	| { kind: "replace"; range: ParsedRange; lineNum: number };
+
+interface Pending {
+	op: PendingOp;
+	payload: string[];
+	pendingBlanks: number;
+}
+
+/**
+ * Token-driven state machine that turns a stream of {@link HashlineToken}s
+ * into the flat list of {@link HashlineEdit}s applied downstream by the
+ * apply/diff layers.
+ *
+ * The executor owns:
+ *   - the running edit index (kept monotonic across pending flushes),
+ *   - the pending-payload buffer (lines accumulated for the most recently
+ *     opened insert/replace op),
+ *   - all parse-time diagnostics (range order, "delete with payload",
+ *     orphan payload, unrecognized op),
+ *   - the {@link terminated} flag set by `envelope-end`/`abort`.
+ *
+ * Tokens are dispatched in the order they arrive; the matching tokenizer
+ * supplies the line numbers carried inside each token so diagnostics line
+ * up with the source.
+ */
+export class HashlineExecutor {
+	#edits: HashlineEdit[] = [];
+	#warnings: string[] = [];
+	#editIndex = 0;
+	#pending: Pending | undefined;
+	#terminated = false;
+
+	/** True once an `envelope-end` or `abort` token has been observed. */
+	get terminated(): boolean {
+		return this.#terminated;
+	}
+
+	/**
+	 * Consume one token. After `terminated` flips true subsequent feeds
+	 * are silently ignored so callers can keep draining their tokenizer
+	 * without explicit early-exit guards.
+	 */
+	feed(token: HashlineToken): void {
+		if (this.#terminated) return;
+
+		switch (token.kind) {
+			case "envelope-begin":
+				return;
+			case "envelope-end":
+				this.#terminated = true;
+				return;
+			case "abort":
+				this.#warnings.push(ABORT_WARNING);
+				this.#terminated = true;
+				return;
+			case "header":
+				this.#flushPending(false);
+				return;
+			case "blank":
+				if (this.#pending) this.#pending.pendingBlanks++;
+				return;
+			case "payload":
+				this.#handlePayload(token.text, token.lineNum);
+				return;
+			case "op-delete":
+				this.#flushPending(false);
+				if (token.trailingPayload) {
+					throw new Error(
+						`line ${token.lineNum}: ${HL_OP_DELETE} deletes only. Payload is forbidden after ${HL_OP_DELETE}; use ${HL_OP_REPLACE} to replace.`,
+					);
+				}
+				validateRangeOrder(token.range, token.lineNum);
+				for (const anchor of expandRange(token.range)) {
+					this.#edits.push({ kind: "delete", anchor, lineNum: token.lineNum, index: this.#editIndex++ });
+				}
+				return;
+			case "op-insert":
+				this.#flushPending(false);
+				this.#pending = {
+					op: { kind: "insert", cursor: token.cursor, lineNum: token.lineNum },
+					payload: [token.inlineBody ?? ""],
+					pendingBlanks: 0,
+				};
+				return;
+			case "op-replace":
+				this.#flushPending(false);
+				validateRangeOrder(token.range, token.lineNum);
+				this.#pending = {
+					op: { kind: "replace", range: token.range, lineNum: token.lineNum },
+					payload: [token.inlineBody ?? ""],
+					pendingBlanks: 0,
+				};
+				return;
+		}
+	}
+
+	/**
+	 * Flush any open pending op (including its trailing blank lines, which
+	 * are payload-significant) and return the accumulated edits and
+	 * warnings. The executor is single-use; reset() is required for reuse.
+	 */
+	end(): { edits: HashlineEdit[]; warnings: string[] } {
+		this.#flushPending(true);
+		return { edits: this.#edits, warnings: this.#warnings };
+	}
+
+	/** Reset to a fresh state so the same instance can drive another parse. */
+	reset(): void {
+		this.#edits = [];
+		this.#warnings = [];
+		this.#editIndex = 0;
+		this.#pending = undefined;
+		this.#terminated = false;
+	}
+
+	#handlePayload(text: string, lineNum: number): void {
+		if (this.#pending) {
+			this.#flushPendingBlanks();
+			this.#pending.payload.push(text);
+			return;
+		}
+
+		// Whitespace-only payload outside any pending op is a visual
+		// separator (matches the legacy outer-loop isBlankLine skip);
+		// only fully-empty lines arrive as `blank` tokens.
+		if (text.trim().length === 0) return;
+		// Orphan payload outside any pending op: pick the most specific
+		// diagnostic so the model sees the actionable hint.
+		if (isDeleteOpWithPayload(text)) {
+			throw new Error(
+				`line ${lineNum}: ${HL_OP_DELETE} deletes only. Payload is forbidden after ${HL_OP_DELETE}; use ${HL_OP_REPLACE} to replace.`,
+			);
+		}
+
+		const firstChar = text[0];
+		const startsWithOp = firstChar !== undefined && HL_OP_CHARS.includes(firstChar);
+		if (startsWithOp || firstChar === "-" || firstChar === "@" || firstChar === "«" || firstChar === "»") {
+			throw new Error(
+				`line ${lineNum}: unrecognized op. Use LINE${HL_OP_INSERT_BEFORE} (insert before), LINE${HL_OP_INSERT_AFTER} (insert after), LINE${HL_OP_REPLACE} / A-B${HL_OP_REPLACE} (replace), or LINE${HL_OP_DELETE} / A-B${HL_OP_DELETE} (delete). ` +
+					`Got ${JSON.stringify(text)}.`,
+			);
+		}
+
+		throw new Error(
+			`line ${lineNum}: payload line has no preceding ${HL_OP_INSERT_BEFORE}, ${HL_OP_INSERT_AFTER}, ${HL_OP_REPLACE}, or ${HL_OP_DELETE} operation. ` +
+				`Got ${JSON.stringify(text)}.`,
+		);
+	}
+
+	#flushPendingBlanks(): void {
+		if (!this.#pending) return;
+		for (let count = 0; count < this.#pending.pendingBlanks; count++) this.#pending.payload.push("");
+		this.#pending.pendingBlanks = 0;
+	}
+
+	#flushPending(includeTrailingBlanks: boolean): void {
+		const pending = this.#pending;
+		if (!pending) return;
+		if (includeTrailingBlanks) this.#flushPendingBlanks();
+
+		const { op, payload } = pending;
+		const linesToInsert = payload;
+
+		if (op.kind === "insert") {
+			for (const text of linesToInsert) {
+				this.#edits.push({
+					kind: "insert",
+					cursor: cloneCursor(op.cursor),
+					text,
+					lineNum: op.lineNum,
+					index: this.#editIndex++,
+				});
+			}
+		} else {
+			for (const text of linesToInsert) {
+				this.#edits.push({
+					kind: "insert",
+					cursor: { kind: "before_anchor", anchor: { ...op.range.start } },
+					text,
+					lineNum: op.lineNum,
+					index: this.#editIndex++,
+				});
+			}
+			for (const anchor of expandRange(op.range)) {
+				this.#edits.push({ kind: "delete", anchor, lineNum: op.lineNum, index: this.#editIndex++ });
+			}
+		}
+
+		this.#pending = undefined;
+	}
+}
+
+/**
+ * Drive a full hashline diff through the tokenizer + executor pipeline and
+ * return the resulting edits plus any parse-time warnings. This is the
+ * convenience entry point most callers want; reach for {@link
+ * HashlineTokenizer}/{@link HashlineExecutor} directly only when you need
+ * streaming feeds, cross-section state, or custom token handling.
+ */
+export function parseHashline(diff: string): { edits: HashlineEdit[]; warnings: string[] } {
+	const tokenizer = new HashlineTokenizer();
+	const executor = new HashlineExecutor();
+	const drain = (tokens: HashlineToken[]): void => {
+		for (const token of tokens) {
+			if (executor.terminated) return;
+			executor.feed(token);
+		}
+	};
+	drain(tokenizer.feed(diff));
+	drain(tokenizer.end());
+	return executor.end();
+}

package/src/hashline/grammar.lark

@@ -3,19 +3,21 @@ begin_patch: "*** Begin Patch" LF
 end_patch: "*** End Patch" LF?
 
 hunk: update_hunk
-update_hunk: "$HFILE$" filename LF line_op*
+update_hunk: "$HFILE$" filename ("#" file_hash)? LF line_op*
 
-filename: /(.+)/
+filename: /([^\s#]+)/
+file_hash: /[0-9a-f]{4}/
 
-line_op: insert_before | insert_after | replace | blank
-insert_before: "$HOP_INSERT_BEFORE$" anchor LF payload+
-insert_after: "$HOP_INSERT_AFTER$" anchor LF payload+
-replace: "$HOP_REPLACE$" range LF payload*
-payload: /[^$HOP_CHARS$$HFILE$\n][^\n]*/ LF | LF
-blank: LF
+line_op: insert_before | insert_after | replace | delete
+insert_before: anchor "$HOP_INSERT_BEFORE$" inline_body? LF payload*
+insert_after: anchor "$HOP_INSERT_AFTER$" inline_body? LF payload*
+replace: range "$HOP_REPLACE$" inline_body? LF payload*
+delete: range "$HOP_DELETE$" LF
+inline_body: /[^\n]+/
+payload: /(.*)/ LF
 
 anchor: LID | "EOF" | "BOF"
-range: LID (".." LID)?
-LID: /[1-9]\d*$HFMT$/
+range: LID ("-" LID)?
+LID: /[1-9]\d*/
 
 %import common.LF

package/src/hashline/hash.ts

@@ -3,70 +3,54 @@
  * and prompt helpers.
  */
 
-import bigrams from "./bigrams.json" with { type: "json" };
-
-/**
- * 647 single-token BPE bigrams for hashline anchors. Every entry tokenizes as
- * exactly one token in modern BPE vocabularies (cl100k / o200k / Claude family),
- * so a hashline anchor built from one bigram is exactly 1 token.
- *
- * This is the complete set of 2-letter lowercase combinations that are single
- * tokens — the 29 missing combinations are rare-letter pairs (q/x/z heavy)
- * that no major BPE vocabulary merges into a single token.
- *
- * Order is stable forever — changing it would invalidate every saved
- * `LINE+ID` reference in transcripts and prompts.
- */
-export const HL_BIGRAMS: readonly string[] = bigrams;
-
-export const HL_BIGRAMS_COUNT = HL_BIGRAMS.length;
+const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 
 /**
- * Decoration prefix that may precede a `LINE+HASH` anchor in tool output:
+ * Decoration prefix that may precede a line number in tool output:
  * `>` (context line in grep), `+` (added line in diff), `-` (removed line),
  * `*` (match line). Any combination, in any order, surrounded by optional
- * whitespace. Output formatters emit at most one decoration per anchor; the
- * regex stays liberal because anchor-ref parsers accept whatever the model
- * echoes back.
+ * whitespace. Output formatters emit at most one decoration per line; the
+ * parser stays liberal because it accepts whatever the model echoes back.
  */
 export const HL_ANCHOR_DECORATION_RE_RAW = `\\s*[>+\\-*]*\\s*`;
 
-/**
- * Capture-group regex source for a decorated `LINE+HASH` anchor. Group 1
- * captures the line number (digits only); group 2 captures the hash. The
- * source is intentionally unanchored — anchoring with `^` (or composing into a
- * larger pattern) is the caller's responsibility.
- */
-export const HL_ANCHOR_RE_RAW = `${HL_ANCHOR_DECORATION_RE_RAW}(\\d+)([a-z]{2})`;
+/** Capture-group regex source for a decorated bare line-number anchor. */
+export const HL_ANCHOR_RE_RAW = `${HL_ANCHOR_DECORATION_RE_RAW}(\\d+)`;
 
-/**
- * Bare `LINE+HASH` Lid (no decorations, no captures, no anchors). Use for
- * embedding inside larger patterns where the line+hash unit appears as a
- * literal (e.g. range bounds, alternation arms, op-line heuristics).
- */
-export const HL_HASH_RE_RAW = `[1-9]\\d*[a-z]{2}`;
+/** Bare positive line-number Lid (no decorations, no captures, no anchors). */
+export const HL_LINE_RE_RAW = `[1-9]\\d*`;
 
-/**
- * Capture-group form of {@link HL_HASH_RE_RAW}: group 1 captures the
- * line number, group 2 captures the hash.
- */
-export const HL_HASH_CAPTURE_RE_RAW = `([1-9]\\d*)([a-z]{2})`;
+/** Capture-group form of {@link HL_LINE_RE_RAW}. */
+export const HL_LINE_CAPTURE_RE_RAW = `([1-9]\\d*)`;
+
+/** Four-hex-character file hash carried by a hashline section header. */
+export const HL_FILE_HASH_RE_RAW = `[0-9a-f]{4}`;
+
+/** Capture-group form of {@link HL_FILE_HASH_RE_RAW}. */
+export const HL_FILE_HASH_CAPTURE_RE_RAW = `(${HL_FILE_HASH_RE_RAW})`;
+
+/** Separator between a hashline file path and its file hash. */
+export const HL_FILE_HASH_SEP = "#";
+
+/** Separator between a line number and displayed line content in hashline mode. */
+export const HL_LINE_BODY_SEP = ":";
 
-/** Width of a hash in display characters. */
-export const HL_HASH_WIDTH = 2;
+/** Regex-escaped form of {@link HL_LINE_BODY_SEP}, safe for embedding inside a regex. */
+export const HL_LINE_BODY_SEP_RE_RAW = regexEscape(HL_LINE_BODY_SEP);
 
 /**
- * Representative hash suffixes for use in user-facing error messages and
- * prompt examples.
+ * Representative file hashes for use in user-facing error messages and prompt
+ * examples.
  */
-export const HL_HASH_EXAMPLES = ["sr", "ab", "th"] as const;
+export const HL_FILE_HASH_EXAMPLES = ["1a2b", "3c4d", "9f3e"] as const;
 
 /**
  * Format a comma-separated list of example anchors with an optional line-number
- * prefix, quoted for inclusion in error messages: `"160sr", "160ab", "160th"`.
+ * prefix, quoted for inclusion in error messages: `"160", "42", "7"`.
  */
 export function describeAnchorExamples(linePrefix = ""): string {
-	return HL_HASH_EXAMPLES.map(e => `"${linePrefix}${e}"`).join(", ");
+	const examples = linePrefix ? [linePrefix, `${linePrefix.slice(0, -1) || "4"}2`, "7"] : ["160", "42", "7"];
+	return examples.map(e => `"${e}"`).join(", ");
 }
 
 /**
@@ -76,98 +60,69 @@ export function describeAnchorExamples(linePrefix = ""): string {
  */
 export function resolveHashlineGrammarPlaceholders(grammar: string): string {
 	return grammar
-		.replaceAll("$HFMT$", "[a-z]{2}")
+		.replaceAll("$HFMT$", "")
+		.replaceAll("$HFILE_HASH$", HL_FILE_HASH_RE_RAW)
+		.replaceAll("$HFILE_HASH_SEP$", HL_FILE_HASH_SEP)
 		.replaceAll("$HOP_INSERT_BEFORE$", HL_OP_INSERT_BEFORE)
 		.replaceAll("$HOP_INSERT_AFTER$", HL_OP_INSERT_AFTER)
 		.replaceAll("$HOP_REPLACE$", HL_OP_REPLACE)
+		.replaceAll("$HOP_DELETE$", HL_OP_DELETE)
 		.replaceAll("$HOP_CHARS$", HL_OP_CHARS)
 		.replaceAll("$HFILE$", HL_FILE_PREFIX);
 }
 
-/** @deprecated Use {@link resolveHashlineGrammarPlaceholders}. */
-export const resolveLarkLidPlaceholders = resolveHashlineGrammarPlaceholders;
-
-const regexEscape = (str: string): string => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-
 /**
- * Hashline edit input markers. File section headers start with {@link HL_FILE_PREFIX};
- * op lines start with a direction/action sigil: {@link HL_OP_INSERT_BEFORE},
- * {@link HL_OP_INSERT_AFTER}, or {@link HL_OP_REPLACE}. Payload lines are
- * verbatim file content and have no per-line marker.
+ * op lines have an `ANCHOR<SIGIL>[INLINE_PAYLOAD]` shape, where SIGIL is one of
+ * {@link HL_OP_INSERT_BEFORE}, {@link HL_OP_INSERT_AFTER}, {@link HL_OP_REPLACE},
+ * or {@link HL_OP_DELETE}.
+ * Multi-line payloads follow on subsequent lines as verbatim file content with no
+ * per-line marker.
  *
  * These constants are the single source of truth for the edit parser, grammar,
  * renderer, and prompt.
  */
-export const HL_OP_INSERT_BEFORE = "«";
-export const HL_OP_INSERT_AFTER = "»";
-export const HL_OP_REPLACE = "≔";
+export const HL_OP_INSERT_BEFORE = "↑";
+export const HL_OP_INSERT_AFTER = "↓";
+export const HL_OP_REPLACE = ":";
+export const HL_OP_DELETE = "!";
 
 /** All hashline edit op sigils, concatenated for fast membership tests. */
-export const HL_OP_CHARS = `${HL_OP_INSERT_BEFORE}${HL_OP_INSERT_AFTER}${HL_OP_REPLACE}`;
+export const HL_OP_CHARS = `${HL_OP_INSERT_BEFORE}${HL_OP_INSERT_AFTER}${HL_OP_REPLACE}${HL_OP_DELETE}`;
 
 /** Hashline edit file section header marker. */
-export const HL_FILE_PREFIX = "§";
-
-/** Stable separator for read/search/hashline display output. Intentionally not configurable. */
-export const HL_BODY_SEP = "|";
-
-/** Regex-escaped form of {@link HL_BODY_SEP}, safe for embedding inside a regex. */
-export const HL_BODY_SEP_RE_RAW = regexEscape(HL_BODY_SEP);
+export const HL_FILE_PREFIX = "¶";
+
+function normalizeFileHashText(text: string): string {
+	return text
+		.replace(/\r/g, "")
+		.split("\n")
+		.map(line => line.trimEnd())
+		.join("\n");
+}
 
 /**
- * Compute a 2-character hash of a single line via xxHash32 mod 647 over
- * {@link HL_BIGRAMS}. The hash depends only on the line's content (after
- * stripping CR and trailing whitespace); the `idx` parameter is accepted
- * for call-site symmetry with line numbers but is intentionally unused so
- * that anchors remain stable across line shifts caused by sibling edits.
- *
- * The line input should not include a trailing newline.
+ * Compute the 4-hex-character hash carried by a hashline section header.
+ * The hash normalizes CR characters and trailing whitespace before hashing so
+ * platform line endings and display-trimmed lines do not invalidate anchors.
  */
-export function computeLineHash(idx: number, line: string): string {
-	void idx;
-	line = line.replace(/\r/g, "").trimEnd();
-	// Seed is fixed so the hash depends only on line content. Earlier we mixed
-	// in `idx` for blank/punctuation-only lines, but that meant any line shift
-	// (e.g. from a sibling edit in the same batch) invalidated anchors whose
-	// content had not changed. Identical blank lines are intentionally allowed
-	// to collide — the edit op's line number disambiguates them.
-	return HL_BIGRAMS[Bun.hash.xxHash32(line, 0) % HL_BIGRAMS_COUNT];
+export function computeFileHash(text: string): string {
+	const normalized = normalizeFileHashText(text);
+	const low16 = Bun.hash.xxHash32(normalized, 0) & 0xffff;
+	return low16.toString(16).padStart(4, "0");
 }
 
-/**
- * Formats an anchor reference given a line number and its text.
- * Returns `LINE+ID` (e.g., `42sr`) — no separator between
- * number and hash.
- */
-export function formatLineHash(line: number, lines: string): string {
-	return `${line}${computeLineHash(line, lines)}`;
+/** Format a hashline section header for a file path and file hash. */
+export function formatHashlineHeader(filePath: string, fileHash: string): string {
+	return `${HL_FILE_PREFIX}${filePath}${HL_FILE_HASH_SEP}${fileHash}`;
 }
 
-/**
- * Formats a single line with a hashline anchor.
- * Returns `LINE+ID|TEXT` (e.g., `42sr|function hi() {`, `3ab|}`).
- */
-export function formatHashLine(lineNumber: number, line: string): string {
-	return `${lineNumber}${computeLineHash(lineNumber, line)}${HL_BODY_SEP}${line}`;
+/** Formats a single numbered line as `LINE:TEXT`. */
+export function formatNumberedLine(lineNumber: number, line: string): string {
+	return `${lineNumber}${HL_LINE_BODY_SEP}${line}`;
 }
 
-/**
- * Format file text with hashline prefixes for display.
- *
- * Each line becomes `LINE+ID|TEXT` where LINENUM is 1-indexed.
- * No padding on line numbers; pipe separator between anchor and content.
- *
- * @param text - Raw file text string
- * @param startLine - First line number (1-indexed, defaults to 1)
- * @returns Formatted string with one hashline-prefixed line per input line
- *
- * @example
- * ```
- * formatHashLines("function hi() {\n  return;\n}")
- * // "1bm|function hi() {\n2er|  return;\n3ab|}"
- * ```
- */
-export function formatHashLines(text: string, startLine = 1): string {
+/** Format file text with hashline-mode line-number prefixes for display. */
+export function formatNumberedLines(text: string, startLine = 1): string {
 	const lines = text.split("\n");
-	return lines.map((line, i) => formatHashLine(startLine + i, line)).join("\n");
+	return lines.map((line, i) => formatNumberedLine(startLine + i, line)).join("\n");
 }

package/src/hashline/index.ts

@@ -4,10 +4,11 @@ export * from "./constants";
 export * from "./diff";
 export * from "./diff-preview";
 export * from "./execute";
+export * from "./executor";
 export * from "./hash";
 export * from "./input";
-export * from "./parser";
 export * from "./prefixes";
 export * from "./recovery";
 export * from "./stream";
+export * from "./tokenizer";
 export * from "./types";