@joshuafolkken/kit 1.1514.0 → 1.2114.0

package/scripts/lib/token-shape.ts

@@ -0,0 +1,111 @@
+// cspell:words xoxb xoxp xapp xoxa xoxo xoxr
+import { json_value } from './json-value'
+
+// The credential shapes a body kit sends out is refused for, and the run board hides. **One list, read
+// in-process**: every outbound body passes through here, so a regex costs microseconds where a
+// secretlint spawn per send would cost a second.
+//
+// Each shape with a `secretlint` rule matches what that preset rule matches — the preset bundles its
+// rules and exports none of their patterns, so each is restated here — and `token-shape.test.ts` runs
+// secretlint over every shape's sample, so a pattern that drifts from the preset fails there. A shape
+// with no rule is kit's own.
+
+interface TokenShape {
+	name: string
+	pattern: RegExp
+	// The preset rule that flags the same shape; `undefined` for one only kit knows.
+	secretlint: string | undefined
+}
+
+const REDACTED = '[REDACTED]'
+const GITHUB_RULE = '@secretlint/secretlint-rule-github'
+
+// `claude setup-token`'s output: no preset rule knows it, and `claude-token-output.ts` captures it.
+const ANTHROPIC_OAUTH_TOKEN = /sk-ant-oat\d+-[\w-]+/u
+
+const SHAPES: ReadonlyArray<TokenShape> = [
+	{
+		name: 'GitHub token',
+		pattern: /(?<!\p{L})(?:ghp|gho|ghu|ghs|ghr)_\w{36}(?!\w)/u,
+		secretlint: GITHUB_RULE,
+	},
+	{
+		name: 'GitHub App installation token',
+		pattern: /(?<!\p{L})ghs_\d{1,20}_[\w-]{1,1000}\.[\w-]{1,10000}\.[\w-]{1,1000}(?![\w-])/u,
+		secretlint: GITHUB_RULE,
+	},
+	{
+		name: 'GitHub fine-grained token',
+		pattern: /(?<!\p{L})github_pat_\w{82}(?!\w)/u,
+		secretlint: GITHUB_RULE,
+	},
+	{
+		name: 'Slack token',
+		pattern:
+			/\b(?:xoxb|xoxp|xapp|xoxa|xoxo|xoxr)-(?:\d-)?(?:[a-zA-Z0-9]{1,40}-)+[a-zA-Z0-9]{1,40}\b/u,
+		secretlint: '@secretlint/secretlint-rule-slack',
+	},
+	{
+		name: 'Anthropic API key',
+		pattern: /(?<!\p{L})sk-ant-api0\d-[\w-]{90,128}AA(?![\w-])/u,
+		secretlint: '@secretlint/secretlint-rule-anthropic',
+	},
+	{ name: 'Anthropic OAuth token', pattern: ANTHROPIC_OAUTH_TOKEN, secretlint: undefined },
+]
+
+const EVERY_SHAPE = new RegExp(SHAPES.map((shape) => shape.pattern.source).join('|'), 'gu')
+
+// The first shape `text` holds, `undefined` for none.
+function find(text: string): TokenShape | undefined {
+	return SHAPES.find((shape) => shape.pattern.test(text))
+}
+
+// Whether `text` holds any shape — `find`'s test without the shape it names.
+function holds(text: string): boolean {
+	return SHAPES.some((shape) => shape.pattern.test(text))
+}
+
+// Throws when `text` holds a credential, naming the shape and never the value: `subject` is what was
+// about to be sent, so the refusal says which send stopped.
+function refuse(text: string, subject: string): void {
+	const shape = find(text)
+	if (shape === undefined) return
+
+	throw new Error(
+		`${subject} holds a ${shape.name}-shaped string, so it was not sent. Remove the credential and send it again.`,
+	)
+}
+
+function redact(text: string): string {
+	return text.replaceAll(EVERY_SHAPE, () => REDACTED)
+}
+
+// A JSON-encoded body puts a token starting a line after the `n` of an escaped newline, and the
+// shapes' "no letter before it" guard reads that `n` as a letter. So a body is scanned as the strings
+// its JSON decodes to — what GitHub stores — and as raw text only when it is not JSON.
+function string_leaves(value: unknown): Array<string> {
+	if (typeof value === 'string') return [value]
+	if (typeof value !== 'object' || value === null) return []
+
+	return Object.values(value).flatMap((entry: unknown) => string_leaves(entry))
+}
+
+function decoded(body: string): string {
+	const value = json_value.parse_or_undefined(body)
+
+	return value === undefined ? body : string_leaves(value).join('\n')
+}
+
+const token_shape = {
+	ANTHROPIC_OAUTH_TOKEN,
+	REDACTED,
+	SHAPES,
+	decoded,
+	find,
+	holds,
+	refuse,
+	redact,
+}
+
+export { token_shape }
+export type { TokenShape }

package/scripts/lib/transcript-record.ts

@@ -0,0 +1,91 @@
+import { z } from 'zod'
+import { json_value } from './json-value'
+
+// One line of a Claude Code JSONL log — a session transcript, or the `stream-json` output of
+// `claude -p`, which writes the same `{ type, message: { id, content } }` shape. Every reader of those
+// lines parses through these schemas and takes what it needs from the record; a fix to the shape
+// reaches all of them at once.
+//
+// **Every field is optional, including `type`.** A required field would make one unexpected block
+// fail the whole array it sits in, and with it every other block on that line — an entire assistant
+// turn dropped because one tool wrote a shape no reader had seen. A block that parses to nothing
+// contributes nothing.
+
+const TEXT_TYPE = 'text'
+const TOOL_USE_TYPE = 'tool_use'
+const TOOL_RESULT_TYPE = 'tool_result'
+
+const BLOCK_SCHEMA = z.object({
+	type: z.string().nullish(),
+	text: z.string().nullish(),
+	name: z.string().nullish(),
+	id: z.string().nullish(),
+	tool_use_id: z.string().nullish(),
+	input: z.unknown().nullish(),
+	// Whether the harness wrote a `tool_result` back as a failure — absent on every other block, and
+	// on the results of tools that never report one.
+	is_error: z.boolean().nullish(),
+	// A result's body: a string or a list of blocks. Read as `unknown` because each reader flattens it
+	// its own way, and a narrower type here would drop the whole line over one odd item.
+	content: z.unknown().nullish(),
+})
+
+const CONTENT_SCHEMA = z.union([z.string(), z.array(BLOCK_SCHEMA)])
+// `id` is the assistant *message* the line belongs to; Claude Code writes one line per content block,
+// so a turn that issued two calls is several lines carrying one id.
+const MESSAGE_SCHEMA = z.object({ id: z.string().nullish(), content: CONTENT_SCHEMA.nullish() })
+
+const LINE_SCHEMA = z.object({
+	type: z.string().nullish(),
+	timestamp: z.string().nullish(),
+	gitBranch: z.string().nullish(),
+	// The two fields outside `message` the harness writes an end-of-task notice on. `unknown`, since a
+	// string here would reject every line whose `content` is a list of blocks.
+	content: z.unknown().nullish(),
+	attachment: z.unknown().nullish(),
+	message: MESSAGE_SCHEMA.nullish(),
+})
+
+type TranscriptBlock = z.infer<typeof BLOCK_SCHEMA>
+type TranscriptContent = z.infer<typeof CONTENT_SCHEMA>
+type TranscriptMessage = z.infer<typeof MESSAGE_SCHEMA>
+type TranscriptRecord = z.infer<typeof LINE_SCHEMA>
+type ToolCallBlock = TranscriptBlock & { name: string }
+
+function read(value: unknown): TranscriptRecord | undefined {
+	return LINE_SCHEMA.safeParse(value).data
+}
+
+// A blank line, a line that is not JSON and a line of another shape are one answer: `undefined`.
+function parse(line: string): TranscriptRecord | undefined {
+	return read(json_value.parse_or_undefined(line))
+}
+
+// The line's blocks. A message written as a bare string — a prompt — carries none.
+function blocks_of(record: TranscriptRecord): ReadonlyArray<TranscriptBlock> {
+	const content = record.message?.content
+
+	return Array.isArray(content) ? content : []
+}
+
+function is_tool_call(block: TranscriptBlock): block is ToolCallBlock {
+	return block.type === TOOL_USE_TYPE && typeof block.name === 'string'
+}
+
+// The calls a line made, in order. A `tool_use` block without a name is not a call anything can name.
+function tool_calls_of(record: TranscriptRecord): Array<ToolCallBlock> {
+	return blocks_of(record).filter((block) => is_tool_call(block))
+}
+
+const transcript_record = {
+	TEXT_TYPE,
+	TOOL_USE_TYPE,
+	TOOL_RESULT_TYPE,
+	read,
+	parse,
+	blocks_of,
+	tool_calls_of,
+}
+
+export type { TranscriptBlock, TranscriptContent, TranscriptMessage, TranscriptRecord }
+export { transcript_record }

package/scripts/lines/effective-limit.ts

@@ -1,6 +1,7 @@
 import { createRequire } from 'node:module'
 import path from 'node:path'
 import { pathToFileURL } from 'node:url'
+import { PACKAGE_JSON } from '#scripts/lib/package-manifest'
 import { z } from 'zod'
 
 // Resolves the `max-lines` entry the project's own eslint applies, so `pnpm josh lines` explains its
@@ -28,7 +29,6 @@ import { z } from 'zod'
 // consumer's config, never by whichever copy happens to sit beside this file.
 
 const ESLINT_MODULE = 'eslint'
-const PACKAGE_JSON = 'package.json'
 const MAX_LINES_RULE = 'max-lines'
 // A rule turned off enforces nothing, so there is no limit to report and no headroom to subtract from.
 // Both spellings occur: `calculateConfigForFile` normalizes severity to a number, but a config read by

package/scripts/lint/eslint-cache-dependencies.ts

@@ -0,0 +1,145 @@
+import { rmSync, writeFileSync } from 'node:fs'
+import path from 'node:path'
+import { file_reader } from '#scripts/lib/read-file'
+import { parse, stringify } from 'flatted'
+import { import_graph, type ImportGraph } from './import-graph'
+
+// eslint's content cache keys a verdict on the linted file's own content, so a file whose imports
+// changed underneath it keeps answering from the cache: the error a type-aware rule would now report
+// — an import that became `any`, read by `no-unsafe-assignment` — never reaches the report, and the
+// type check does not report the `no-unsafe-*` rules either (joshuafolkken/kit#4415). CI restores
+// the gate's cache file, so it answers the same stale green.
+//
+// So each cached entry also records what its verdict read: the hash of every file its imports reach
+// (`import-graph.ts`). Before a cached run, an entry whose imports no longer hash the same is dropped,
+// and eslint lints that file afresh; after it, the entries eslint wrote are given the hash taken
+// before the run, which is the state of the files that run read. An entry with no hash — one an
+// unguarded run wrote, or one from before this — is dropped as well, because nothing says what it
+// read.
+//
+// The drop clears a stale red as well as a stale green: an import rule's report — `import/no-cycle`,
+// `import-x/named` — is about another file too, and once that file changes the entry is linted
+// afresh. Only imports are followed, so a rule that reads the files importing this one — a
+// reverse-dependency rule such as `import/no-unused-modules`, a no-op under ESLint 10 — would go
+// stale here. This is the one place cache staleness is decided; it replaced the cross-file re-read that
+// once ran after a red cached run (joshuafolkken/kit#3641, #3667, #4407, #4420).
+//
+// The record rides inside eslint's own cache file, in the entry's `data` beside the results eslint
+// keeps there, rather than a file of its own: it then travels wherever the cache file does — a lane's
+// seed, CI's cache — and eslint replaces `data` whole whenever it lints the file again, so a hash can
+// never outlive the verdict it was taken for. The file is `flat-cache`'s, serialized with `flatted`.
+
+type CacheMeta = Record<string, unknown>
+
+interface CacheEntry {
+	key: string
+	value: CacheMeta
+}
+
+type Hashes = ReadonlyMap<string, string>
+
+function is_record(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function is_entry(item: unknown): item is CacheEntry {
+	if (!is_record(item)) return false
+	const { key, value } = item
+
+	return typeof key === 'string' && is_record(value)
+}
+
+function is_entries(value: unknown): value is Array<CacheEntry> {
+	return Array.isArray(value) && value.every((item: unknown) => is_entry(item))
+}
+
+function parsed_or_undefined(content: string): unknown {
+	try {
+		return parse(content)
+	} catch {
+		return undefined
+	}
+}
+
+// Undefined for a missing file and for one this cannot read as eslint writes it.
+function read_entries(cache_file: string): Array<CacheEntry> | undefined {
+	const content = file_reader.read_optional(cache_file)
+	if (content === undefined) return undefined
+	const parsed = parsed_or_undefined(content)
+
+	return is_entries(parsed) ? parsed : undefined
+}
+
+// A cache file this cannot read — a corrupt one, or one another layout wrote into a cache lanes share —
+// is removed before the run rather than left to eslint, so the run rebuilds it whole and every entry it
+// writes carries a hash (joshuafolkken/kit#4497).
+function read_or_discard(cache_file: string): Array<CacheEntry> {
+	const entries = read_entries(cache_file)
+
+	if (entries === undefined) rmSync(cache_file, { force: true })
+
+	return entries ?? []
+}
+
+function write_entries(cache_file: string, entries: ReadonlyArray<CacheEntry>): void {
+	writeFileSync(cache_file, stringify(entries))
+}
+
+function recorded_hash(entry: CacheEntry): string | undefined {
+	const { data } = entry.value
+	if (!is_record(data)) return undefined
+	const { dependencies } = data
+
+	return typeof dependencies === 'string' ? dependencies : undefined
+}
+
+// Drops every entry whose imports no longer hash as recorded, and answers the hashes it took.
+function drop_stale(cache_file: string, graph: ImportGraph): Hashes {
+	const entries = read_or_discard(cache_file)
+	const hashes = new Map(entries.map((entry) => [entry.key, graph.closure_hash(entry.key)]))
+	const kept = entries.filter((entry) => recorded_hash(entry) === hashes.get(entry.key))
+
+	if (kept.length !== entries.length) write_entries(cache_file, kept)
+
+	return hashes
+}
+
+// eslint writes `data` only for a file it linted, so an entry with `data` and no hash is one this run
+// linted. A file the run linted for the first time had no hash taken before it, so it is taken now.
+function record_entry(entry: CacheEntry, hashes: Hashes, graph: ImportGraph): boolean {
+	const { data } = entry.value
+	if (!is_record(data) || 'dependencies' in data) return false
+
+	Object.assign(data, { dependencies: hashes.get(entry.key) ?? graph.closure_hash(entry.key) })
+
+	return true
+}
+
+function record(cache_file: string, hashes: Hashes, graph: ImportGraph): void {
+	const entries = read_entries(cache_file)
+	if (entries === undefined) return
+	const recorded = entries.filter((entry) => record_entry(entry, hashes, graph))
+
+	if (recorded.length > 0) write_entries(cache_file, entries)
+}
+
+// Runs one cached eslint run between the drop and the record. A project without `typescript` or a `tsconfig.json` runs it as it was.
+async function guard<T>(
+	cache_file: string,
+	run: () => Promise<T>,
+	root: string = process.cwd(),
+): Promise<T> {
+	const graph = await import_graph.create(root)
+	if (graph === undefined) return await run()
+	const location = path.resolve(root, cache_file)
+	const hashes = drop_stale(location, graph)
+	const result = await run()
+
+	record(location, hashes, graph)
+
+	return result
+}
+
+const eslint_cache_dependencies = { guard }
+
+export { eslint_cache_dependencies }

package/scripts/lint/eslint-daemon-sources.ts

@@ -0,0 +1,79 @@
+import { createHash } from 'node:crypto'
+import { readFileSync } from 'node:fs'
+import path from 'node:path'
+import { git_spawn_sync } from '#scripts/git/git-spawn-sync'
+
+// **What the daemon's type program may hold an old copy of: every file whose content differs from the
+// commit checked out.** The program keeps each source it read but did not lint, and never re-reads it,
+// so a shell edit to an imported module — one the edit hook never sees — left typed rules failing on
+// the module's old exports, and the cache guard then recorded that verdict under the new closure hash,
+// so it replayed even after the daemon was stopped (joshuafolkken/kit#4867). A file equal to HEAD is
+// already covered by the HEAD in the fingerprint, so only the changed ones are hashed: one entry each,
+// `<path>\t<content digest>`, one per line, sorted.
+//
+// The edit hook lints the file it edited through the daemon, which reads it fresh, so it refreshes
+// that one entry (`refresh`); an edit it never saw leaves the entry behind, and the next run restarts.
+
+// NUL-separated, so a path git would quote and escape — any non-ASCII name — reads as the file it is.
+const CHANGED_FILES: ReadonlyArray<string> = ['diff', 'HEAD', '--name-only', '--relative', '-z']
+const NEW_FILES: ReadonlyArray<string> = ['ls-files', '--others', '--exclude-standard', '-z']
+const PATH_SEPARATOR = '\0'
+const LINE_SEPARATOR = '\n'
+const ENTRY_SEPARATOR = '\t'
+const DIGEST_ALGORITHM = 'sha256'
+const ABSENT = 'absent'
+const UNREAD = ':unread'
+
+// One order for `snapshot` and `refresh` alike, so a refreshed record compares equal to a fresh one.
+function by_text(left: string, right: string): number {
+	return left.localeCompare(right)
+}
+
+function lines_of(text: string): ReadonlyArray<string> {
+	return text.split(LINE_SEPARATOR).filter((line) => line !== '')
+}
+
+function content_digest(file: string): string {
+	try {
+		return createHash(DIGEST_ALGORITHM).update(readFileSync(file)).digest('hex')
+	} catch {
+		return ABSENT
+	}
+}
+
+// A git that could not answer lists a path no file has, so its record differs from one with no changes.
+function changed_paths(project_root: string): ReadonlyArray<string> {
+	const listed = [CHANGED_FILES, NEW_FILES].map(
+		(git_arguments) => git_spawn_sync.read(git_arguments, { cwd: project_root }) ?? UNREAD,
+	)
+
+	const paths = listed.join(PATH_SEPARATOR).split(PATH_SEPARATOR)
+
+	return [...new Set(paths.filter((relative) => relative !== ''))]
+}
+
+// The changed files' entries, sorted — nothing when no file differs from HEAD.
+function snapshot(project_root: string): string {
+	return changed_paths(project_root)
+		.map(
+			(relative) =>
+				`${relative}${ENTRY_SEPARATOR}${content_digest(path.join(project_root, relative))}`,
+		)
+		.toSorted(by_text)
+		.join(LINE_SEPARATOR)
+}
+
+// A recorded fingerprint — its first line the daemon's other inputs, the rest `snapshot`'s entries —
+// with `relative`'s entry as `current` has it now: the daemon has just read that one file fresh.
+function refresh(recorded: string, current: string, relative: string): string {
+	const prefix = `${relative}${ENTRY_SEPARATOR}`
+	const [inputs = '', ...entries] = lines_of(recorded)
+	const others = entries.filter((line) => !line.startsWith(prefix))
+	const fresh = lines_of(current).filter((line) => line.startsWith(prefix))
+
+	return [inputs, ...[...others, ...fresh].toSorted(by_text)].join(LINE_SEPARATOR)
+}
+
+const eslint_daemon_sources = { refresh, snapshot, LINE_SEPARATOR }
+
+export { eslint_daemon_sources }