@payconductor/houndsman 0.3.0 → 0.5.0

package/src/server/client/utils/code-highlight.ts

@@ -0,0 +1,422 @@
+/**
+ * Destaque de sintaxe de código (cliente) — implementação própria e leve.
+ *
+ * A linguagem é inferida pela extensão do arquivo do finding. Para extensões
+ * conhecidas destacamos strings, comentários, números, keywords, tipos,
+ * funções e propriedades; para as demais, o evidence continua texto puro.
+ */
+
+import { highlightJson } from "./json-highlight";
+
+export type CodeTokenType =
+	| "plain"
+	| "keyword"
+	| "string"
+	| "comment"
+	| "number"
+	| "type"
+	| "function"
+	| "property"
+	| "punct";
+
+export type CodeToken = { text: string; type: CodeTokenType };
+
+export type HighlightedCode = {
+	language: string;
+	label: string;
+	tokens: CodeToken[];
+};
+
+type LanguageConfig = {
+	label: string;
+	keywords: ReadonlySet<string>;
+	types?: ReadonlySet<string>;
+	lineComments: readonly string[];
+	blockComments: readonly (readonly [string, string])[];
+	/** Templates strings com backtick (JS/TS/Go). */
+	backtick?: boolean;
+	/** Strings de três aspas (Python). */
+	tripleQuotes?: boolean;
+	/** `chave:` no início da linha vira propriedade (YAML). */
+	keyColon?: boolean;
+	/** Identificador seguido de `:` vira propriedade (CSS). */
+	cssProperty?: boolean;
+	/** Rastreia tags/atributos (HTML/XML). */
+	html?: boolean;
+};
+
+function words(value: string): ReadonlySet<string> {
+	return new Set(value.split(/\s+/).filter(Boolean));
+}
+
+const JS_KEYWORDS = words(
+	"as async await break case catch class const continue debugger declare default delete do else enum export extends false finally for from function get if implements import in infer instanceof interface is keyof let module namespace new null of private protected public readonly return satisfies set static super switch this throw true try type typeof undefined var void while with yield",
+);
+
+const CONFIG: Record<string, LanguageConfig> = {
+	typescript: {
+		label: "TypeScript",
+		keywords: JS_KEYWORDS,
+		types: words("any bigint boolean never number object string symbol unknown"),
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+		backtick: true,
+	},
+	javascript: {
+		label: "JavaScript",
+		keywords: JS_KEYWORDS,
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+		backtick: true,
+	},
+	python: {
+		label: "Python",
+		keywords: words(
+			"and as assert async await break class continue def del elif else except False finally for from global if import in is lambda None nonlocal not or pass raise return True try while with yield",
+		),
+		lineComments: ["#"],
+		blockComments: [],
+		tripleQuotes: true,
+	},
+	go: {
+		label: "Go",
+		keywords: words(
+			"break case chan const continue default defer else fallthrough for func go goto if import interface map package range return select struct switch type var nil true false",
+		),
+		types: words(
+			"bool byte complex64 complex128 error float32 float64 int int8 int16 int32 int64 rune string uint uint8 uint16 uint32 uint64 uintptr",
+		),
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+		backtick: true,
+	},
+	rust: {
+		label: "Rust",
+		keywords: words(
+			"as async await break const continue crate dyn else enum extern false fn for if impl in let loop match mod move mut pub ref return self Self static struct super trait true type unsafe use where while",
+		),
+		types: words(
+			"bool char f32 f64 i8 i16 i32 i64 i128 isize str String u8 u16 u32 u64 u128 usize",
+		),
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+	},
+	java: {
+		label: "Java",
+		keywords: words(
+			"abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for goto if implements import instanceof int interface long native new null package private protected public record return sealed short static strictfp super switch synchronized this throw throws transient true false try var void volatile while",
+		),
+		types: words("boolean byte char double float int long short String void"),
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+	},
+	csharp: {
+		label: "C#",
+		keywords: words(
+			"abstract as async await base bool break byte case catch char checked class const continue decimal default delegate do double else enum event explicit extern false finally fixed float for foreach goto if implicit in int interface internal is lock long namespace new null object operator out override params private protected public readonly record ref return sbyte sealed short sizeof stackalloc static string struct switch this throw true try typeof uint ulong unchecked unsafe ushort using var virtual void volatile while",
+		),
+		types: words(
+			"bool byte char decimal double float int long object sbyte short string uint ulong ushort void",
+		),
+		lineComments: ["//"],
+		blockComments: [["/*", "*/"]],
+	},
+	php: {
+		label: "PHP",
+		keywords: words(
+			"abstract and array as break callable case catch class clone const continue declare default do echo else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval exit extends final finally fn for foreach function global goto if implements include include_once instanceof insteadof interface isset list match namespace new null or print private protected public readonly require require_once return static switch throw trait true false try unset use var while xor yield",
+		),
+		lineComments: ["//", "#"],
+		blockComments: [["/*", "*/"]],
+	},
+	ruby: {
+		label: "Ruby",
+		keywords: words(
+			"alias and begin break case class def defined? do else elsif end ensure false for if in module next nil not or redo rescue retry return self super then true undef unless until when while yield",
+		),
+		lineComments: ["#"],
+		blockComments: [],
+	},
+	bash: {
+		label: "Shell",
+		keywords: words(
+			"if then else elif fi for in do done while until case esac function select time local export readonly return break continue true false",
+		),
+		lineComments: ["#"],
+		blockComments: [],
+	},
+	sql: {
+		label: "SQL",
+		keywords: words(
+			"select from where join inner left right full outer on group by order having limit offset insert into values update set delete create table alter drop index view as and or not null is in like between exists case when then else end distinct union all count sum avg min max returning with asc desc",
+		),
+		lineComments: ["--"],
+		blockComments: [["/*", "*/"]],
+	},
+	yaml: {
+		label: "YAML",
+		keywords: words("true false null yes no on off"),
+		lineComments: ["#"],
+		blockComments: [],
+		keyColon: true,
+	},
+	css: {
+		label: "CSS",
+		keywords: words(
+			"important media import supports keyframes font-face from to and or not only",
+		),
+		lineComments: [],
+		blockComments: [["/*", "*/"]],
+		cssProperty: true,
+	},
+	html: {
+		label: "HTML",
+		keywords: new Set(),
+		lineComments: [],
+		blockComments: [["<!--", "-->"]],
+		html: true,
+	},
+};
+
+const EXTENSION_TO_LANGUAGE: Record<string, string> = {
+	ts: "typescript",
+	tsx: "typescript",
+	mts: "typescript",
+	cts: "typescript",
+	js: "javascript",
+	jsx: "javascript",
+	mjs: "javascript",
+	cjs: "javascript",
+	json: "json",
+	py: "python",
+	go: "go",
+	rs: "rust",
+	java: "java",
+	cs: "csharp",
+	php: "php",
+	rb: "ruby",
+	sh: "bash",
+	bash: "bash",
+	zsh: "bash",
+	sql: "sql",
+	yml: "yaml",
+	yaml: "yaml",
+	css: "css",
+	scss: "css",
+	html: "html",
+	htm: "html",
+	xml: "html",
+};
+
+/** Linguagem a partir da extensão do arquivo (null = sem highlight). */
+export function languageFromFile(file: string): string | null {
+	const base = file.split(/[\\/]/).pop() ?? file;
+	const dot = base.lastIndexOf(".");
+	if (dot <= 0 || dot === base.length - 1) return null;
+	return EXTENSION_TO_LANGUAGE[base.slice(dot + 1).toLowerCase()] ?? null;
+}
+
+/** Linguagem a partir do nome/alias usado em fences markdown (`ts`, `py`…). */
+export function languageFromName(name: string | null | undefined): string | null {
+	if (!name) return null;
+	const key = name.trim().toLowerCase();
+	const alias: Record<string, string> = {
+		ts: "typescript",
+		tsx: "typescript",
+		js: "javascript",
+		jsx: "javascript",
+		py: "python",
+		rs: "rust",
+		cs: "csharp",
+		"c#": "csharp",
+		rb: "ruby",
+		sh: "bash",
+		shell: "bash",
+		yml: "yaml",
+		scss: "css",
+		xml: "html",
+	};
+	const language = alias[key] ?? key;
+	return language === "json" || CONFIG[language] ? language : null;
+}
+
+/** Destaca o código; null quando a linguagem é desconhecida/ausente. */
+export function highlightCode(
+	code: string,
+	language: string | null,
+): HighlightedCode | null {
+	if (!language) return null;
+	if (language === "json") {
+		return {
+			language,
+			label: "JSON",
+			tokens: highlightJson(code).map((token) => ({
+				text: token.text,
+				type:
+					token.type === "key"
+						? "property"
+						: token.type === "literal"
+							? "keyword"
+							: token.type,
+			})),
+		};
+	}
+	const config = CONFIG[language];
+	if (!config) return null;
+	return {
+		language,
+		label: config.label,
+		tokens: tokenize(code, config),
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Tokenização
+// ---------------------------------------------------------------------------
+
+function matchComment(rest: string, config: LanguageConfig): string | null {
+	for (const [start, end] of config.blockComments) {
+		if (!rest.startsWith(start)) continue;
+		const close = rest.indexOf(end, start.length);
+		return close < 0 ? rest : rest.slice(0, close + end.length);
+	}
+	for (const marker of config.lineComments) {
+		if (!rest.startsWith(marker)) continue;
+		const newline = rest.indexOf("\n");
+		return newline < 0 ? rest : rest.slice(0, newline);
+	}
+	return null;
+}
+
+function previousNonSpace(code: string, index: number): string {
+	for (let i = index - 1; i >= 0; i--) {
+		if (!/\s/.test(code[i])) return code[i];
+	}
+	return "";
+}
+
+function nextNonSpace(code: string, index: number): string {
+	for (let i = index; i < code.length; i++) {
+		if (!/\s/.test(code[i])) return code[i];
+	}
+	return "";
+}
+
+function atLineStart(code: string, index: number): boolean {
+	const lineStart = code.lastIndexOf("\n", index - 1) + 1;
+	return /^\s*$/.test(code.slice(lineStart, index));
+}
+
+function tokenize(code: string, config: LanguageConfig): CodeToken[] {
+	const tokens: CodeToken[] = [];
+	let index = 0;
+	let inTag = false;
+	const push = (text: string, type: CodeTokenType): void => {
+		if (text.length > 0) tokens.push({ text, type });
+	};
+
+	while (index < code.length) {
+		const rest = code.slice(index);
+
+		const comment = matchComment(rest, config);
+		if (comment !== null) {
+			push(comment, "comment");
+			index += comment.length;
+			continue;
+		}
+
+		if (config.tripleQuotes && (rest.startsWith('"""') || rest.startsWith("'''"))) {
+			const quote = rest.slice(0, 3);
+			const close = code.indexOf(quote, index + 3);
+			const end = close < 0 ? code.length : close + 3;
+			push(code.slice(index, end), "string");
+			index = end;
+			continue;
+		}
+
+		const char = code[index];
+		if (char === '"' || char === "'" || (char === "`" && config.backtick)) {
+			let end = index + 1;
+			while (end < code.length) {
+				if (code[end] === "\\" && char !== "`") {
+					end += 2;
+					continue;
+				}
+				if (code[end] === char) {
+					end++;
+					break;
+				}
+				end++;
+			}
+			push(code.slice(index, end), "string");
+			index = end;
+			continue;
+		}
+
+		const number = /^(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(rest);
+		if (number) {
+			push(number[0], "number");
+			index += number[0].length;
+			continue;
+		}
+
+		const identifier = /^[A-Za-z_$][\w$]*/.exec(rest);
+		if (identifier) {
+			const word = identifier[0];
+			const previous = previousNonSpace(code, index);
+			const next = nextNonSpace(code, index + word.length);
+			let type: CodeTokenType = "plain";
+			if (config.keywords.has(word)) {
+				type = "keyword";
+			} else if (config.types?.has(word)) {
+				type = "type";
+			} else if (next === "(") {
+				type = "function";
+			} else if (config.html && inTag) {
+				type =
+					previous === "<" || (previous === "/" && code[index - 2] === "<")
+						? "keyword"
+						: "property";
+			} else if (config.keyColon && next === ":" && atLineStart(code, index)) {
+				type = "property";
+			} else if (config.cssProperty && next === ":") {
+				type = "property";
+			} else if (previous === ".") {
+				type = "property";
+			} else if (/^[A-Z]/.test(word)) {
+				type = "type";
+			}
+			push(word, type);
+			index += word.length;
+			continue;
+		}
+
+		if (char === "<") {
+			push(char, "punct");
+			inTag = config.html ? true : inTag;
+			index++;
+			continue;
+		}
+		if (char === ">" && inTag) {
+			push(char, "punct");
+			inTag = false;
+			index++;
+			continue;
+		}
+		if (/[{}()[\];,.:=+\-*/%<>!&|?~^@]/.test(char)) {
+			push(char, "punct");
+			index++;
+			continue;
+		}
+		if (/\s/.test(char)) {
+			const whitespace = /^\s+/.exec(rest);
+			const text = whitespace?.[0] ?? char;
+			push(text, "plain");
+			index += text.length;
+			continue;
+		}
+		push(char, "plain");
+		index++;
+	}
+	return tokens;
+}

package/src/server/client/utils/constants.ts

@@ -0,0 +1,226 @@
+/**
+ * Constantes de apresentação do cliente (labels, ordem, emojis).
+ */
+
+import type {
+	ArtifactKind,
+	ArtifactMeta,
+	Decision,
+	ExecState,
+	RunStatus,
+	RunSummary,
+	Severity,
+} from "./types";
+
+/** Rótulo de exibição de cada severidade (o CSS dá a cor via `sev-<nivel>`). */
+export const SEVERITY_LABELS: Record<Severity, string> = {
+	blocking: "Blocking",
+	attention: "Attention",
+	suggestion: "Suggestion",
+	nit: "Nit",
+};
+
+export const SEVERITY_ORDER: Severity[] = ["blocking", "attention", "suggestion", "nit"];
+
+export const DECISIONS: Decision[] = ["pending", "execute", "ignore"];
+
+export const DECISION_LABELS: Record<Decision, string> = {
+	pending: "Pendente",
+	execute: "Executar",
+	ignore: "Ignorar",
+};
+
+export const EXEC_STATES: ExecState[] = ["pending", "in_progress", "done", "stale"];
+
+export const RUN_STATUS_LABELS: Record<RunStatus, string> = {
+	reviewing: "Revisando",
+	review_done: "Revisão pronta",
+	fixing: "Corrigindo",
+	fix_done: "Correção pronta",
+};
+
+/** Limiar de inatividade da correção para exibir timeout (display-only). */
+export const STATUS_TIMEOUT_MS = 10 * 60 * 1000;
+
+/** Revisão despacha subagentes em ondas longas — janela maior que a correção. */
+export const REVIEW_TIMEOUT_MS = 30 * 60 * 1000;
+
+/** Intervalo do polling da UI (o badge conta o tempo desde a última sync). */
+export const AUTO_REFRESH_MS = 10_000;
+
+export type RunStatusView = { kind: string; label: string; timeout: boolean };
+
+/**
+ * View do status de uma run — timeout só existe quando há um heartbeat de
+ * execução (`reviewing`/`fixing`). `updatedAt` muda com triagem/sync e não
+ * serve para provar que a skill está rodando.
+ */
+export function describeRunStatus(
+	run: RunSummary,
+	now: number = Date.now(),
+): RunStatusView | null {
+	const status = run.status;
+	if (!status) return null;
+	const active = status === "reviewing" || status === "fixing";
+	if (!active) {
+		return { kind: status, label: RUN_STATUS_LABELS[status], timeout: false };
+	}
+
+	// Sem heartbeat não há execução conhecida: evita o falso "timeout".
+	const heartbeat = run.heartbeatAt ? Date.parse(run.heartbeatAt) : Number.NaN;
+	if (!Number.isFinite(heartbeat)) return null;
+
+	const timeoutMs = status === "reviewing" ? REVIEW_TIMEOUT_MS : STATUS_TIMEOUT_MS;
+	if (now - heartbeat > timeoutMs) {
+		return {
+			kind: "timeout",
+			label: status === "reviewing" ? "Timeout na revisão" : "Timeout na correção",
+			timeout: true,
+		};
+	}
+	return { kind: status, label: RUN_STATUS_LABELS[status], timeout: false };
+}
+
+/** Ordem canônica dos tipos de artefato na lista da UI. */
+const KIND_ORDER: ArtifactKind[] = [
+	"review",
+	"resume",
+	"domain-review",
+	"verify",
+	"brief",
+	"manifest",
+	"classification",
+	"candidates",
+	"context",
+	"context-brief",
+	"context-summary",
+	"finding-report",
+	"other",
+];
+
+/** Grupo visual de cada tipo (relatórios, briefs, núcleo, contexto…). */
+const GROUP_BY_KIND: Record<ArtifactKind, { key: string; label: string; order: number }> =
+	{
+		review: { key: "report", label: "Relatórios", order: 0 },
+		resume: { key: "report", label: "Relatórios", order: 0 },
+		"domain-review": { key: "report", label: "Relatórios", order: 0 },
+		verify: { key: "verify", label: "Verificação", order: 1 },
+		brief: { key: "brief", label: "Briefs (auditoria)", order: 2 },
+		manifest: { key: "core", label: "Núcleo da revisão", order: 3 },
+		classification: { key: "core", label: "Núcleo da revisão", order: 3 },
+		candidates: { key: "core", label: "Núcleo da revisão", order: 3 },
+		context: { key: "context", label: "Contexto da PR", order: 4 },
+		"context-brief": { key: "context", label: "Contexto da PR", order: 4 },
+		"context-summary": { key: "context", label: "Contexto da PR", order: 4 },
+		"finding-report": { key: "other", label: "Outros", order: 5 },
+		other: { key: "other", label: "Outros", order: 5 },
+	};
+
+/** Rótulo curto do tipo (badge do item). */
+export const ARTIFACT_KIND_LABELS: Record<ArtifactKind, string> = {
+	review: "relatório",
+	resume: "resumo",
+	"domain-review": "domínio",
+	verify: "verify",
+	brief: "brief",
+	manifest: "manifesto",
+	classification: "classificação",
+	candidates: "jev",
+	context: "contexto",
+	"context-brief": "brief ctx",
+	"context-summary": "resumo ctx",
+	"finding-report": "findings",
+	other: "outro",
+};
+
+/** Nome exibido (briefs perdem o prefixo `briefs/`). */
+export function displayArtifactName(meta: ArtifactMeta): string {
+	return meta.name.startsWith("briefs/")
+		? meta.name.slice("briefs/".length)
+		: meta.name;
+}
+
+/** Descrição curta do artefato (cabeçalho do painel). */
+export function describeArtifact(meta: ArtifactMeta): string {
+	switch (meta.kind) {
+		case "review":
+			return "Relatório consolidado da revisão — o entregável.";
+		case "resume":
+			return "Resumo final da revisão: visão de negócio e pontos técnicos diretos (leitura humana).";
+		case "domain-review":
+			return "Relatório isolado do domínio: achados daquele watchdog.";
+		case "verify":
+			return "Resultado do `findings verify` (resolvidos, persistentes e incertos).";
+		case "brief":
+			return "Instruções enviadas ao subagente (auditoria do prompt).";
+		case "manifest":
+			return "Resumo da execução: watchdogs, arquivos, orçamento, churn, Jev e contexto.";
+		case "classification":
+			return "Classificação do diff: watchdogs ativados por arquivo, status (A/M/D/R) e churn.";
+		case "candidates":
+			return "Indícios da pré-análise Jev (regra + arquivo + confiança) — não são achados.";
+		case "context":
+			return "Payload bruto da PR coletado via `gh` (ou marcador de indisponível).";
+		case "context-brief":
+			return "Brief do subagente que resume o contexto da PR.";
+		case "context-summary":
+			return "Resumo da PR (objetivo, motivação e alterações) gerado no Passo 0.";
+		default:
+			return meta.name;
+	}
+}
+
+/** `true` se o conteúdo deve ser renderizado com destaque de JSON. */
+export function isJsonArtifact(meta: ArtifactMeta | undefined): boolean {
+	if (!meta) return false;
+	return meta.contentType.includes("json") || meta.name.toLowerCase().endsWith(".json");
+}
+
+/** `true` se o conteúdo deve ser renderizado como markdown. */
+export function isMarkdownArtifact(meta: ArtifactMeta | undefined): boolean {
+	if (!meta) return false;
+	return (
+		meta.contentType.includes("markdown") || meta.name.toLowerCase().endsWith(".md")
+	);
+}
+
+/** Agrupa os artefatos para a navegação lateral (ordem estável). */
+export function groupArtifacts(
+	items: ArtifactMeta[],
+): { key: string; label: string; items: ArtifactMeta[] }[] {
+	const groups = new Map<
+		string,
+		{ key: string; label: string; order: number; items: ArtifactMeta[] }
+	>();
+	for (const item of items) {
+		const meta = GROUP_BY_KIND[item.kind] ?? GROUP_BY_KIND.other;
+		const group = groups.get(meta.key) ?? {
+			key: meta.key,
+			label: meta.label,
+			order: meta.order,
+			items: [],
+		};
+		group.items.push(item);
+		groups.set(meta.key, group);
+	}
+	return [...groups.values()]
+		.sort((a, b) => a.order - b.order)
+		.map((group) => ({
+			key: group.key,
+			label: group.label,
+			items: group.items.sort(
+				(a, b) =>
+					KIND_ORDER.indexOf(a.kind) - KIND_ORDER.indexOf(b.kind) ||
+					a.name.localeCompare(b.name),
+			),
+		}));
+}
+
+/** Legenda única de cada aba (explica o que a aba é; mostrada quando selecionada). */
+export const TAB_CAPTIONS = {
+	resume: "Resumo principal da revisão: o que muda em termos de negócio, números diretos (arquivos, churn, domínios) e os pontos técnicos que exigem ação.",
+	findings:
+		"Triagem dos achados da revisão: decida Executar, Ignorar ou deixe Pendente — o comentário orienta o auto-fix. Agente: houndsman findings list --decision pending --json · findings done <id>.",
+	artifacts:
+		"Artefatos persistidos da run: relatórios, briefs de auditoria, manifesto, classificação, Jev, contexto da PR e verificação. JSON com destaque de sintaxe; markdown renderizado com highlight de código.",
+} as const;

package/src/server/client/utils/json-highlight.ts

@@ -0,0 +1,55 @@
+/**
+ * Destaque de sintaxe JSON (cliente) — implementação própria e leve.
+ *
+ * Não puxa lib de highlight: tokeniza o texto já formatado e devolve spans.
+ * Funciona para qualquer JSON gerado pelo houndsman (manifest, classification,
+ * context, candidates, …).
+ */
+
+export type JsonTokenType = "plain" | "key" | "string" | "number" | "literal" | "punct";
+
+export type JsonToken = { text: string; type: JsonTokenType };
+
+const TOKEN =
+	/("(?:\\.|[^"\\])*")(\s*:)?|(-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)|\b(true|false|null)\b|([{}[\],:])/g;
+
+/** Pretty-print best-effort; devolve o original se não for JSON válido. */
+export function formatJson(text: string): string {
+	try {
+		return JSON.stringify(JSON.parse(text), null, 2);
+	} catch {
+		return text;
+	}
+}
+
+/** Tokeniza o JSON preservando o texto original (inclusive espaços/quebras). */
+export function highlightJson(text: string): JsonToken[] {
+	const tokens: JsonToken[] = [];
+	let last = 0;
+	for (const match of text.matchAll(TOKEN)) {
+		const index = match.index ?? 0;
+		if (index > last) {
+			tokens.push({ text: text.slice(last, index), type: "plain" });
+		}
+		if (match[1] !== undefined) {
+			tokens.push({
+				text: match[1],
+				type: match[2] !== undefined ? "key" : "string",
+			});
+			if (match[2] !== undefined) {
+				tokens.push({ text: match[2], type: "punct" });
+			}
+		} else if (match[3] !== undefined) {
+			tokens.push({ text: match[3], type: "number" });
+		} else if (match[4] !== undefined) {
+			tokens.push({ text: match[4], type: "literal" });
+		} else if (match[5] !== undefined) {
+			tokens.push({ text: match[5], type: "punct" });
+		}
+		last = index + match[0].length;
+	}
+	if (last < text.length) {
+		tokens.push({ text: text.slice(last), type: "plain" });
+	}
+	return tokens;
+}