@payconductor/houndsman 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -6
- package/package.json +1 -1
- package/src/cli/commands/context-command.ts +11 -9
- package/src/cli/commands/doctor-command.ts +5 -0
- package/src/cli/commands/findings-command.ts +13 -3
- package/src/cli/commands/review-command.ts +1 -2
- package/src/cli/help.ts +7 -3
- package/src/cli/index.ts +2 -2
- package/src/core/commands/context.ts +29 -13
- package/src/core/commands/findings.ts +155 -120
- package/src/core/commands/review.ts +7 -8
- package/src/core/commands/scaffold.ts +15 -16
- package/src/core/config.ts +16 -7
- package/src/core/engine/houndsman-engine.ts +15 -1
- package/src/core/engine/review-session.ts +37 -9
- package/src/core/findings-state.ts +32 -71
- package/src/core/jev/jev-candidates.ts +49 -35
- package/src/core/orchestrator.ts +175 -128
- package/src/core/paths.ts +23 -17
- package/src/core/pr-context.ts +19 -5
- package/src/core/project-config.ts +4 -5
- package/src/core/renderers/manifest-writer.ts +29 -18
- package/src/core/run-layout.ts +4 -104
- package/src/core/run-status.ts +28 -53
- package/src/core/services/git-repository.ts +16 -0
- package/src/core/services/project-files.ts +13 -1
- package/src/core/storage/factory.ts +24 -0
- package/src/core/storage/file/file-storage.ts +269 -0
- package/src/core/storage/paths.ts +64 -0
- package/src/core/storage/repo-identity.ts +70 -0
- package/src/core/storage/sqlite/mappers.ts +160 -0
- package/src/core/storage/sqlite/schema.ts +108 -0
- package/src/core/storage/sqlite/sqlite-storage.ts +427 -0
- package/src/core/storage/types.ts +143 -0
- package/src/core/storage/work-area.ts +191 -0
- package/src/index.ts +27 -6
- package/src/server/artifacts.ts +8 -61
- package/src/server/client/ArtifactsPanel.tsx +137 -23
- package/src/server/client/FindingCard.tsx +37 -6
- package/src/server/client/MarkdownText.tsx +101 -0
- package/src/server/client/SubmitModal.tsx +9 -4
- package/src/server/client/code-highlight.ts +422 -0
- package/src/server/client/constants.ts +130 -24
- package/src/server/client/json-highlight.ts +55 -0
- package/src/server/client/markdown.ts +153 -0
- package/src/server/client/types.ts +23 -0
- package/src/server/index.ts +95 -15
- package/src/server/run-repository.ts +144 -136
- package/src/skill/OUTPUT-FORMAT.md +6 -4
- package/src/skill/README.md +22 -14
- package/src/skill/SETUP.md +4 -4
- package/src/skill/fix.md +5 -5
- package/src/skill/hunt.md +27 -18
- package/src/skill/train.md +1 -1
- package/src/skill/verify.md +5 -5
- package/src/tests/__snapshots__/baseline.test.ts.snap +44 -33
- package/src/tests/baseline.test.ts +76 -34
- package/src/tests/cli.test.ts +40 -19
- package/src/tests/code-highlight.test.ts +123 -0
- package/src/tests/domain.test.ts +1 -1
- package/src/tests/findings-commands.test.ts +153 -131
- package/src/tests/findings-state.test.ts +6 -3
- package/src/tests/fixtures/fake-project.ts +0 -1
- package/src/tests/infrastructure.test.ts +33 -38
- package/src/tests/markdown.test.ts +89 -0
- package/src/tests/review-scripts.test.ts +8 -5
- package/src/tests/run-status.test.ts +33 -23
- package/src/tests/server.test.ts +108 -75
- package/src/tests/storage.test.ts +386 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Render de markdown mínimo para o campo `action` do finding.
|
|
3
|
+
*
|
|
4
|
+
* Sem HTML cru: tudo vira nós do Preact. Blocos de código reutilizam o
|
|
5
|
+
* highlight de código; links passam por `safeUrl`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { useMemo } from "preact/hooks";
|
|
9
|
+
import { highlightCode, languageFromName } from "./code-highlight";
|
|
10
|
+
import { type InlineToken, parseInline, parseMarkdown, safeUrl } from "./markdown";
|
|
11
|
+
|
|
12
|
+
function renderInlineToken(token: InlineToken) {
|
|
13
|
+
switch (token.type) {
|
|
14
|
+
case "code":
|
|
15
|
+
return <code class="md-code">{token.text}</code>;
|
|
16
|
+
case "strong":
|
|
17
|
+
return <strong>{token.text}</strong>;
|
|
18
|
+
case "em":
|
|
19
|
+
return <em>{token.text}</em>;
|
|
20
|
+
case "link": {
|
|
21
|
+
const url = safeUrl(token.url);
|
|
22
|
+
return url ? (
|
|
23
|
+
<a href={url} target="_blank" rel="noreferrer">
|
|
24
|
+
{token.text}
|
|
25
|
+
</a>
|
|
26
|
+
) : (
|
|
27
|
+
token.text
|
|
28
|
+
);
|
|
29
|
+
}
|
|
30
|
+
default:
|
|
31
|
+
return token.text;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function InlineTokens({ text }: { text: string }) {
|
|
36
|
+
const tokens = useMemo(() => parseInline(text), [text]);
|
|
37
|
+
return <>{tokens.map((token) => renderInlineToken(token))}</>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function CodeBlock({ code, language }: { code: string; language: string | null }) {
|
|
41
|
+
const highlighted = useMemo(
|
|
42
|
+
() => highlightCode(code, languageFromName(language)),
|
|
43
|
+
[code, language],
|
|
44
|
+
);
|
|
45
|
+
return (
|
|
46
|
+
<pre class="md-code-block">
|
|
47
|
+
{highlighted
|
|
48
|
+
? highlighted.tokens.map((token) => (
|
|
49
|
+
<span
|
|
50
|
+
class={
|
|
51
|
+
token.type === "plain" ? undefined : `tok-${token.type}`
|
|
52
|
+
}
|
|
53
|
+
>
|
|
54
|
+
{token.text}
|
|
55
|
+
</span>
|
|
56
|
+
))
|
|
57
|
+
: code}
|
|
58
|
+
</pre>
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Renderiza o markdown do action preservando quebras de linha e listas. */
|
|
63
|
+
export function MarkdownText({ text }: { text: string }) {
|
|
64
|
+
const blocks = useMemo(() => parseMarkdown(text), [text]);
|
|
65
|
+
return (
|
|
66
|
+
<>
|
|
67
|
+
{blocks.map((block) => {
|
|
68
|
+
switch (block.type) {
|
|
69
|
+
case "paragraph":
|
|
70
|
+
return (
|
|
71
|
+
<p class="md-p">
|
|
72
|
+
<InlineTokens text={block.text} />
|
|
73
|
+
</p>
|
|
74
|
+
);
|
|
75
|
+
case "list": {
|
|
76
|
+
const items = block.items.map((item) => (
|
|
77
|
+
<li>
|
|
78
|
+
<InlineTokens text={item} />
|
|
79
|
+
</li>
|
|
80
|
+
));
|
|
81
|
+
return block.ordered ? (
|
|
82
|
+
<ol class="md-list">{items}</ol>
|
|
83
|
+
) : (
|
|
84
|
+
<ul class="md-list">{items}</ul>
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
case "code":
|
|
88
|
+
return <CodeBlock code={block.code} language={block.language} />;
|
|
89
|
+
case "quote":
|
|
90
|
+
return (
|
|
91
|
+
<blockquote class="md-quote">
|
|
92
|
+
<InlineTokens text={block.text} />
|
|
93
|
+
</blockquote>
|
|
94
|
+
);
|
|
95
|
+
default:
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
})}
|
|
99
|
+
</>
|
|
100
|
+
);
|
|
101
|
+
}
|
|
@@ -122,8 +122,13 @@ export function SubmitModal({
|
|
|
122
122
|
() => selected.filter((candidate) => includeSent || !candidate.sent),
|
|
123
123
|
[selected, includeSent],
|
|
124
124
|
);
|
|
125
|
-
const
|
|
126
|
-
|
|
125
|
+
const totals = useMemo(
|
|
126
|
+
() => ({
|
|
127
|
+
line: toSend.filter((candidate) => candidate.kind === "line").length,
|
|
128
|
+
file: toSend.filter((candidate) => candidate.kind === "file").length,
|
|
129
|
+
}),
|
|
130
|
+
[toSend],
|
|
131
|
+
);
|
|
127
132
|
const skipped = selected.length - toSend.length;
|
|
128
133
|
|
|
129
134
|
const submit = useCallback(async () => {
|
|
@@ -277,8 +282,8 @@ export function SubmitModal({
|
|
|
277
282
|
|
|
278
283
|
<div class="submit-summary">
|
|
279
284
|
Serão disparados <b>{toSend.length}</b> comentário(s)
|
|
280
|
-
— <b>{
|
|
281
|
-
arquivo
|
|
285
|
+
— <b>{totals.line}</b> na linha · <b>{totals.file}</b>{" "}
|
|
286
|
+
no arquivo
|
|
282
287
|
{skipped > 0 ? (
|
|
283
288
|
<>
|
|
284
289
|
{" "}
|
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Destaque de sintaxe de código (cliente) — implementação própria e leve.
|
|
3
|
+
*
|
|
4
|
+
* A linguagem é inferida pela extensão do arquivo do finding. Para extensões
|
|
5
|
+
* conhecidas destacamos strings, comentários, números, keywords, tipos,
|
|
6
|
+
* funções e propriedades; para as demais, o evidence continua texto puro.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { highlightJson } from "./json-highlight";
|
|
10
|
+
|
|
11
|
+
export type CodeTokenType =
|
|
12
|
+
| "plain"
|
|
13
|
+
| "keyword"
|
|
14
|
+
| "string"
|
|
15
|
+
| "comment"
|
|
16
|
+
| "number"
|
|
17
|
+
| "type"
|
|
18
|
+
| "function"
|
|
19
|
+
| "property"
|
|
20
|
+
| "punct";
|
|
21
|
+
|
|
22
|
+
export type CodeToken = { text: string; type: CodeTokenType };
|
|
23
|
+
|
|
24
|
+
export type HighlightedCode = {
|
|
25
|
+
language: string;
|
|
26
|
+
label: string;
|
|
27
|
+
tokens: CodeToken[];
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
type LanguageConfig = {
|
|
31
|
+
label: string;
|
|
32
|
+
keywords: ReadonlySet<string>;
|
|
33
|
+
types?: ReadonlySet<string>;
|
|
34
|
+
lineComments: readonly string[];
|
|
35
|
+
blockComments: readonly (readonly [string, string])[];
|
|
36
|
+
/** Templates strings com backtick (JS/TS/Go). */
|
|
37
|
+
backtick?: boolean;
|
|
38
|
+
/** Strings de três aspas (Python). */
|
|
39
|
+
tripleQuotes?: boolean;
|
|
40
|
+
/** `chave:` no início da linha vira propriedade (YAML). */
|
|
41
|
+
keyColon?: boolean;
|
|
42
|
+
/** Identificador seguido de `:` vira propriedade (CSS). */
|
|
43
|
+
cssProperty?: boolean;
|
|
44
|
+
/** Rastreia tags/atributos (HTML/XML). */
|
|
45
|
+
html?: boolean;
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
function words(value: string): ReadonlySet<string> {
|
|
49
|
+
return new Set(value.split(/\s+/).filter(Boolean));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const JS_KEYWORDS = words(
|
|
53
|
+
"as async await break case catch class const continue debugger declare default delete do else enum export extends false finally for from function get if implements import in infer instanceof interface is keyof let module namespace new null of private protected public readonly return satisfies set static super switch this throw true try type typeof undefined var void while with yield",
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
const CONFIG: Record<string, LanguageConfig> = {
|
|
57
|
+
typescript: {
|
|
58
|
+
label: "TypeScript",
|
|
59
|
+
keywords: JS_KEYWORDS,
|
|
60
|
+
types: words("any bigint boolean never number object string symbol unknown"),
|
|
61
|
+
lineComments: ["//"],
|
|
62
|
+
blockComments: [["/*", "*/"]],
|
|
63
|
+
backtick: true,
|
|
64
|
+
},
|
|
65
|
+
javascript: {
|
|
66
|
+
label: "JavaScript",
|
|
67
|
+
keywords: JS_KEYWORDS,
|
|
68
|
+
lineComments: ["//"],
|
|
69
|
+
blockComments: [["/*", "*/"]],
|
|
70
|
+
backtick: true,
|
|
71
|
+
},
|
|
72
|
+
python: {
|
|
73
|
+
label: "Python",
|
|
74
|
+
keywords: words(
|
|
75
|
+
"and as assert async await break class continue def del elif else except False finally for from global if import in is lambda None nonlocal not or pass raise return True try while with yield",
|
|
76
|
+
),
|
|
77
|
+
lineComments: ["#"],
|
|
78
|
+
blockComments: [],
|
|
79
|
+
tripleQuotes: true,
|
|
80
|
+
},
|
|
81
|
+
go: {
|
|
82
|
+
label: "Go",
|
|
83
|
+
keywords: words(
|
|
84
|
+
"break case chan const continue default defer else fallthrough for func go goto if import interface map package range return select struct switch type var nil true false",
|
|
85
|
+
),
|
|
86
|
+
types: words(
|
|
87
|
+
"bool byte complex64 complex128 error float32 float64 int int8 int16 int32 int64 rune string uint uint8 uint16 uint32 uint64 uintptr",
|
|
88
|
+
),
|
|
89
|
+
lineComments: ["//"],
|
|
90
|
+
blockComments: [["/*", "*/"]],
|
|
91
|
+
backtick: true,
|
|
92
|
+
},
|
|
93
|
+
rust: {
|
|
94
|
+
label: "Rust",
|
|
95
|
+
keywords: words(
|
|
96
|
+
"as async await break const continue crate dyn else enum extern false fn for if impl in let loop match mod move mut pub ref return self Self static struct super trait true type unsafe use where while",
|
|
97
|
+
),
|
|
98
|
+
types: words(
|
|
99
|
+
"bool char f32 f64 i8 i16 i32 i64 i128 isize str String u8 u16 u32 u64 u128 usize",
|
|
100
|
+
),
|
|
101
|
+
lineComments: ["//"],
|
|
102
|
+
blockComments: [["/*", "*/"]],
|
|
103
|
+
},
|
|
104
|
+
java: {
|
|
105
|
+
label: "Java",
|
|
106
|
+
keywords: words(
|
|
107
|
+
"abstract assert boolean break byte case catch char class const continue default do double else enum extends final finally float for goto if implements import instanceof int interface long native new null package private protected public record return sealed short static strictfp super switch synchronized this throw throws transient true false try var void volatile while",
|
|
108
|
+
),
|
|
109
|
+
types: words("boolean byte char double float int long short String void"),
|
|
110
|
+
lineComments: ["//"],
|
|
111
|
+
blockComments: [["/*", "*/"]],
|
|
112
|
+
},
|
|
113
|
+
csharp: {
|
|
114
|
+
label: "C#",
|
|
115
|
+
keywords: words(
|
|
116
|
+
"abstract as async await base bool break byte case catch char checked class const continue decimal default delegate do double else enum event explicit extern false finally fixed float for foreach goto if implicit in int interface internal is lock long namespace new null object operator out override params private protected public readonly record ref return sbyte sealed short sizeof stackalloc static string struct switch this throw true try typeof uint ulong unchecked unsafe ushort using var virtual void volatile while",
|
|
117
|
+
),
|
|
118
|
+
types: words(
|
|
119
|
+
"bool byte char decimal double float int long object sbyte short string uint ulong ushort void",
|
|
120
|
+
),
|
|
121
|
+
lineComments: ["//"],
|
|
122
|
+
blockComments: [["/*", "*/"]],
|
|
123
|
+
},
|
|
124
|
+
php: {
|
|
125
|
+
label: "PHP",
|
|
126
|
+
keywords: words(
|
|
127
|
+
"abstract and array as break callable case catch class clone const continue declare default do echo else elseif empty enddeclare endfor endforeach endif endswitch endwhile eval exit extends final finally fn for foreach function global goto if implements include include_once instanceof insteadof interface isset list match namespace new null or print private protected public readonly require require_once return static switch throw trait true false try unset use var while xor yield",
|
|
128
|
+
),
|
|
129
|
+
lineComments: ["//", "#"],
|
|
130
|
+
blockComments: [["/*", "*/"]],
|
|
131
|
+
},
|
|
132
|
+
ruby: {
|
|
133
|
+
label: "Ruby",
|
|
134
|
+
keywords: words(
|
|
135
|
+
"alias and begin break case class def defined? do else elsif end ensure false for if in module next nil not or redo rescue retry return self super then true undef unless until when while yield",
|
|
136
|
+
),
|
|
137
|
+
lineComments: ["#"],
|
|
138
|
+
blockComments: [],
|
|
139
|
+
},
|
|
140
|
+
bash: {
|
|
141
|
+
label: "Shell",
|
|
142
|
+
keywords: words(
|
|
143
|
+
"if then else elif fi for in do done while until case esac function select time local export readonly return break continue true false",
|
|
144
|
+
),
|
|
145
|
+
lineComments: ["#"],
|
|
146
|
+
blockComments: [],
|
|
147
|
+
},
|
|
148
|
+
sql: {
|
|
149
|
+
label: "SQL",
|
|
150
|
+
keywords: words(
|
|
151
|
+
"select from where join inner left right full outer on group by order having limit offset insert into values update set delete create table alter drop index view as and or not null is in like between exists case when then else end distinct union all count sum avg min max returning with asc desc",
|
|
152
|
+
),
|
|
153
|
+
lineComments: ["--"],
|
|
154
|
+
blockComments: [["/*", "*/"]],
|
|
155
|
+
},
|
|
156
|
+
yaml: {
|
|
157
|
+
label: "YAML",
|
|
158
|
+
keywords: words("true false null yes no on off"),
|
|
159
|
+
lineComments: ["#"],
|
|
160
|
+
blockComments: [],
|
|
161
|
+
keyColon: true,
|
|
162
|
+
},
|
|
163
|
+
css: {
|
|
164
|
+
label: "CSS",
|
|
165
|
+
keywords: words(
|
|
166
|
+
"important media import supports keyframes font-face from to and or not only",
|
|
167
|
+
),
|
|
168
|
+
lineComments: [],
|
|
169
|
+
blockComments: [["/*", "*/"]],
|
|
170
|
+
cssProperty: true,
|
|
171
|
+
},
|
|
172
|
+
html: {
|
|
173
|
+
label: "HTML",
|
|
174
|
+
keywords: new Set(),
|
|
175
|
+
lineComments: [],
|
|
176
|
+
blockComments: [["<!--", "-->"]],
|
|
177
|
+
html: true,
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
const EXTENSION_TO_LANGUAGE: Record<string, string> = {
|
|
182
|
+
ts: "typescript",
|
|
183
|
+
tsx: "typescript",
|
|
184
|
+
mts: "typescript",
|
|
185
|
+
cts: "typescript",
|
|
186
|
+
js: "javascript",
|
|
187
|
+
jsx: "javascript",
|
|
188
|
+
mjs: "javascript",
|
|
189
|
+
cjs: "javascript",
|
|
190
|
+
json: "json",
|
|
191
|
+
py: "python",
|
|
192
|
+
go: "go",
|
|
193
|
+
rs: "rust",
|
|
194
|
+
java: "java",
|
|
195
|
+
cs: "csharp",
|
|
196
|
+
php: "php",
|
|
197
|
+
rb: "ruby",
|
|
198
|
+
sh: "bash",
|
|
199
|
+
bash: "bash",
|
|
200
|
+
zsh: "bash",
|
|
201
|
+
sql: "sql",
|
|
202
|
+
yml: "yaml",
|
|
203
|
+
yaml: "yaml",
|
|
204
|
+
css: "css",
|
|
205
|
+
scss: "css",
|
|
206
|
+
html: "html",
|
|
207
|
+
htm: "html",
|
|
208
|
+
xml: "html",
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
/** Linguagem a partir da extensão do arquivo (null = sem highlight). */
|
|
212
|
+
export function languageFromFile(file: string): string | null {
|
|
213
|
+
const base = file.split(/[\\/]/).pop() ?? file;
|
|
214
|
+
const dot = base.lastIndexOf(".");
|
|
215
|
+
if (dot <= 0 || dot === base.length - 1) return null;
|
|
216
|
+
return EXTENSION_TO_LANGUAGE[base.slice(dot + 1).toLowerCase()] ?? null;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** Linguagem a partir do nome/alias usado em fences markdown (`ts`, `py`…). */
|
|
220
|
+
export function languageFromName(name: string | null | undefined): string | null {
|
|
221
|
+
if (!name) return null;
|
|
222
|
+
const key = name.trim().toLowerCase();
|
|
223
|
+
const alias: Record<string, string> = {
|
|
224
|
+
ts: "typescript",
|
|
225
|
+
tsx: "typescript",
|
|
226
|
+
js: "javascript",
|
|
227
|
+
jsx: "javascript",
|
|
228
|
+
py: "python",
|
|
229
|
+
rs: "rust",
|
|
230
|
+
cs: "csharp",
|
|
231
|
+
"c#": "csharp",
|
|
232
|
+
rb: "ruby",
|
|
233
|
+
sh: "bash",
|
|
234
|
+
shell: "bash",
|
|
235
|
+
yml: "yaml",
|
|
236
|
+
scss: "css",
|
|
237
|
+
xml: "html",
|
|
238
|
+
};
|
|
239
|
+
const language = alias[key] ?? key;
|
|
240
|
+
return language === "json" || CONFIG[language] ? language : null;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Destaca o código; null quando a linguagem é desconhecida/ausente. */
|
|
244
|
+
export function highlightCode(
|
|
245
|
+
code: string,
|
|
246
|
+
language: string | null,
|
|
247
|
+
): HighlightedCode | null {
|
|
248
|
+
if (!language) return null;
|
|
249
|
+
if (language === "json") {
|
|
250
|
+
return {
|
|
251
|
+
language,
|
|
252
|
+
label: "JSON",
|
|
253
|
+
tokens: highlightJson(code).map((token) => ({
|
|
254
|
+
text: token.text,
|
|
255
|
+
type:
|
|
256
|
+
token.type === "key"
|
|
257
|
+
? "property"
|
|
258
|
+
: token.type === "literal"
|
|
259
|
+
? "keyword"
|
|
260
|
+
: token.type,
|
|
261
|
+
})),
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
const config = CONFIG[language];
|
|
265
|
+
if (!config) return null;
|
|
266
|
+
return {
|
|
267
|
+
language,
|
|
268
|
+
label: config.label,
|
|
269
|
+
tokens: tokenize(code, config),
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
// Tokenização
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
function matchComment(rest: string, config: LanguageConfig): string | null {
|
|
278
|
+
for (const [start, end] of config.blockComments) {
|
|
279
|
+
if (!rest.startsWith(start)) continue;
|
|
280
|
+
const close = rest.indexOf(end, start.length);
|
|
281
|
+
return close < 0 ? rest : rest.slice(0, close + end.length);
|
|
282
|
+
}
|
|
283
|
+
for (const marker of config.lineComments) {
|
|
284
|
+
if (!rest.startsWith(marker)) continue;
|
|
285
|
+
const newline = rest.indexOf("\n");
|
|
286
|
+
return newline < 0 ? rest : rest.slice(0, newline);
|
|
287
|
+
}
|
|
288
|
+
return null;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function previousNonSpace(code: string, index: number): string {
|
|
292
|
+
for (let i = index - 1; i >= 0; i--) {
|
|
293
|
+
if (!/\s/.test(code[i])) return code[i];
|
|
294
|
+
}
|
|
295
|
+
return "";
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function nextNonSpace(code: string, index: number): string {
|
|
299
|
+
for (let i = index; i < code.length; i++) {
|
|
300
|
+
if (!/\s/.test(code[i])) return code[i];
|
|
301
|
+
}
|
|
302
|
+
return "";
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function atLineStart(code: string, index: number): boolean {
|
|
306
|
+
const lineStart = code.lastIndexOf("\n", index - 1) + 1;
|
|
307
|
+
return /^\s*$/.test(code.slice(lineStart, index));
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function tokenize(code: string, config: LanguageConfig): CodeToken[] {
|
|
311
|
+
const tokens: CodeToken[] = [];
|
|
312
|
+
let index = 0;
|
|
313
|
+
let inTag = false;
|
|
314
|
+
const push = (text: string, type: CodeTokenType): void => {
|
|
315
|
+
if (text.length > 0) tokens.push({ text, type });
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
while (index < code.length) {
|
|
319
|
+
const rest = code.slice(index);
|
|
320
|
+
|
|
321
|
+
const comment = matchComment(rest, config);
|
|
322
|
+
if (comment !== null) {
|
|
323
|
+
push(comment, "comment");
|
|
324
|
+
index += comment.length;
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
if (config.tripleQuotes && (rest.startsWith('"""') || rest.startsWith("'''"))) {
|
|
329
|
+
const quote = rest.slice(0, 3);
|
|
330
|
+
const close = code.indexOf(quote, index + 3);
|
|
331
|
+
const end = close < 0 ? code.length : close + 3;
|
|
332
|
+
push(code.slice(index, end), "string");
|
|
333
|
+
index = end;
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
const char = code[index];
|
|
338
|
+
if (char === '"' || char === "'" || (char === "`" && config.backtick)) {
|
|
339
|
+
let end = index + 1;
|
|
340
|
+
while (end < code.length) {
|
|
341
|
+
if (code[end] === "\\" && char !== "`") {
|
|
342
|
+
end += 2;
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
if (code[end] === char) {
|
|
346
|
+
end++;
|
|
347
|
+
break;
|
|
348
|
+
}
|
|
349
|
+
end++;
|
|
350
|
+
}
|
|
351
|
+
push(code.slice(index, end), "string");
|
|
352
|
+
index = end;
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const number = /^(?:0[xX][0-9a-fA-F]+|\d+(?:\.\d+)?(?:[eE][+-]?\d+)?)/.exec(rest);
|
|
357
|
+
if (number) {
|
|
358
|
+
push(number[0], "number");
|
|
359
|
+
index += number[0].length;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const identifier = /^[A-Za-z_$][\w$]*/.exec(rest);
|
|
364
|
+
if (identifier) {
|
|
365
|
+
const word = identifier[0];
|
|
366
|
+
const previous = previousNonSpace(code, index);
|
|
367
|
+
const next = nextNonSpace(code, index + word.length);
|
|
368
|
+
let type: CodeTokenType = "plain";
|
|
369
|
+
if (config.keywords.has(word)) {
|
|
370
|
+
type = "keyword";
|
|
371
|
+
} else if (config.types?.has(word)) {
|
|
372
|
+
type = "type";
|
|
373
|
+
} else if (next === "(") {
|
|
374
|
+
type = "function";
|
|
375
|
+
} else if (config.html && inTag) {
|
|
376
|
+
type =
|
|
377
|
+
previous === "<" || (previous === "/" && code[index - 2] === "<")
|
|
378
|
+
? "keyword"
|
|
379
|
+
: "property";
|
|
380
|
+
} else if (config.keyColon && next === ":" && atLineStart(code, index)) {
|
|
381
|
+
type = "property";
|
|
382
|
+
} else if (config.cssProperty && next === ":") {
|
|
383
|
+
type = "property";
|
|
384
|
+
} else if (previous === ".") {
|
|
385
|
+
type = "property";
|
|
386
|
+
} else if (/^[A-Z]/.test(word)) {
|
|
387
|
+
type = "type";
|
|
388
|
+
}
|
|
389
|
+
push(word, type);
|
|
390
|
+
index += word.length;
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
if (char === "<") {
|
|
395
|
+
push(char, "punct");
|
|
396
|
+
inTag = config.html ? true : inTag;
|
|
397
|
+
index++;
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
if (char === ">" && inTag) {
|
|
401
|
+
push(char, "punct");
|
|
402
|
+
inTag = false;
|
|
403
|
+
index++;
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (/[{}()[\];,.:=+\-*/%<>!&|?~^@]/.test(char)) {
|
|
407
|
+
push(char, "punct");
|
|
408
|
+
index++;
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
if (/\s/.test(char)) {
|
|
412
|
+
const whitespace = /^\s+/.exec(rest);
|
|
413
|
+
const text = whitespace?.[0] ?? char;
|
|
414
|
+
push(text, "plain");
|
|
415
|
+
index += text.length;
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
push(char, "plain");
|
|
419
|
+
index++;
|
|
420
|
+
}
|
|
421
|
+
return tokens;
|
|
422
|
+
}
|