@davidbyoung/software-factory 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +5 -0
- package/LICENSE +21 -0
- package/README.md +135 -2
- package/actions/factory-agent/action.yml +218 -0
- package/agents/factory-test-reviewer.md +129 -0
- package/agents/factory-ticket-readiness.md +107 -0
- package/agents/factory-ui-reviewer.md +160 -0
- package/bin/factory.js +45 -0
- package/bin/launch.js +67 -0
- package/dist/board.d.mts +128 -0
- package/dist/board.mjs +456 -0
- package/dist/build.d.mts +18 -0
- package/dist/build.mjs +60 -0
- package/dist/cli.d.mts +17 -0
- package/dist/cli.mjs +264 -0
- package/dist/code-host/github.d.mts +118 -0
- package/dist/code-host/github.mjs +419 -0
- package/dist/code-host/index.d.mts +9 -0
- package/dist/code-host/index.mjs +18 -0
- package/dist/code-host/types.d.mts +205 -0
- package/dist/code-host/types.mjs +12 -0
- package/dist/config.d.mts +166 -0
- package/dist/config.mjs +548 -0
- package/dist/deps.d.mts +32 -0
- package/dist/deps.mjs +126 -0
- package/dist/doctor.d.mts +34 -0
- package/dist/doctor.mjs +188 -0
- package/dist/guard-commit.d.mts +41 -0
- package/dist/guard-commit.mjs +205 -0
- package/dist/guard-permission.d.mts +15 -0
- package/dist/guard-permission.mjs +52 -0
- package/dist/guard-ticket-text.d.mts +27 -0
- package/dist/guard-ticket-text.mjs +288 -0
- package/dist/guard.d.mts +4 -0
- package/dist/guard.mjs +37 -0
- package/dist/init.d.mts +58 -0
- package/dist/init.mjs +556 -0
- package/dist/lib/clock.d.mts +14 -0
- package/dist/lib/clock.mjs +36 -0
- package/dist/lib/gh.d.mts +28 -0
- package/dist/lib/gh.mjs +92 -0
- package/dist/lib/glob.d.mts +2 -0
- package/dist/lib/glob.mjs +41 -0
- package/dist/lib/pricing.d.mts +25 -0
- package/dist/lib/pricing.mjs +77 -0
- package/dist/lib/usage.d.mts +34 -0
- package/dist/lib/usage.mjs +128 -0
- package/dist/lib/work-status.d.mts +251 -0
- package/dist/lib/work-status.mjs +510 -0
- package/dist/merge-policy.d.mts +66 -0
- package/dist/merge-policy.mjs +146 -0
- package/dist/parts.d.mts +39 -0
- package/dist/parts.mjs +138 -0
- package/dist/poller.d.mts +61 -0
- package/dist/poller.mjs +247 -0
- package/dist/pr-watch.d.mts +45 -0
- package/dist/pr-watch.mjs +244 -0
- package/dist/pr.d.mts +65 -0
- package/dist/pr.mjs +571 -0
- package/dist/review-cost.d.mts +18 -0
- package/dist/review-cost.mjs +95 -0
- package/dist/review-tools.d.mts +5 -0
- package/dist/review-tools.mjs +46 -0
- package/dist/review-verdict.d.mts +50 -0
- package/dist/review-verdict.mjs +171 -0
- package/dist/reviewers.d.mts +54 -0
- package/dist/reviewers.mjs +312 -0
- package/dist/runner/github-actions.d.mts +12 -0
- package/dist/runner/github-actions.mjs +97 -0
- package/dist/runner/index.d.mts +9 -0
- package/dist/runner/index.mjs +22 -0
- package/dist/runner/types.d.mts +60 -0
- package/dist/runner/types.mjs +7 -0
- package/dist/session-start.d.mts +20 -0
- package/dist/session-start.mjs +115 -0
- package/dist/site/capture.d.mts +44 -0
- package/dist/site/capture.mjs +114 -0
- package/dist/site/measure.d.mts +83 -0
- package/dist/site/measure.mjs +225 -0
- package/dist/site/pages.d.mts +10 -0
- package/dist/site/pages.mjs +71 -0
- package/dist/site/pixels.d.mts +47 -0
- package/dist/site/pixels.mjs +157 -0
- package/dist/site/servers.d.mts +47 -0
- package/dist/site/servers.mjs +141 -0
- package/dist/site.d.mts +60 -0
- package/dist/site.mjs +341 -0
- package/dist/status.d.mts +100 -0
- package/dist/status.mjs +514 -0
- package/dist/sync.d.mts +59 -0
- package/dist/sync.mjs +292 -0
- package/dist/ticket.d.mts +61 -0
- package/dist/ticket.mjs +562 -0
- package/dist/tickets/github.d.mts +128 -0
- package/dist/tickets/github.mjs +491 -0
- package/dist/tickets/index.d.mts +15 -0
- package/dist/tickets/index.mjs +27 -0
- package/dist/tickets/transport.d.mts +42 -0
- package/dist/tickets/transport.mjs +158 -0
- package/dist/tickets/types.d.mts +191 -0
- package/dist/tickets/types.mjs +76 -0
- package/docs/builds.md +227 -0
- package/docs/cli.md +221 -0
- package/docs/configuration.md +809 -0
- package/docs/design.md +106 -0
- package/docs/developing.md +400 -0
- package/docs/existing-repo.md +138 -0
- package/docs/github/reference.md +288 -0
- package/docs/github/setup.md +312 -0
- package/docs/glossary.md +106 -0
- package/docs/limitations.md +115 -0
- package/docs/plugins.md +192 -0
- package/docs/quick-start.md +222 -0
- package/docs/removing.md +76 -0
- package/docs/skills.md +270 -0
- package/docs/updating.md +108 -0
- package/docs/workflow.md +195 -0
- package/package.json +67 -4
- package/poller.Dockerfile +21 -0
- package/review.md +170 -0
- package/rules.md +313 -0
- package/settings.schema.json +411 -0
- package/skills/factory-address-review/SKILL.md +547 -0
- package/skills/factory-address-review/patch-coverage.py +68 -0
- package/skills/factory-build/SKILL.md +412 -0
- package/skills/factory-deps/SKILL.md +185 -0
- package/skills/factory-design/SKILL.md +159 -0
- package/skills/factory-diagnose/SKILL.md +93 -0
- package/skills/factory-mutate/SKILL.md +76 -0
- package/skills/factory-refine/SKILL.md +417 -0
- package/skills/factory-resolve-conflicts/SKILL.md +235 -0
- package/workflows/factory-board-sync.yml +91 -0
- package/workflows/factory-build.yml +72 -0
- package/workflows/factory-check.yml +37 -0
- package/workflows/factory-claude-review.yml +133 -0
- package/workflows/factory-conflicts.yml +69 -0
- package/workflows/factory-deps-auto-merge.yml +43 -0
- package/workflows/factory-deps.yml +59 -0
- package/workflows/factory-merge-policy.yml +121 -0
- package/workflows/factory-review-verdict.yml +54 -0
- package/workflows/factory-review.yml +75 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/** The axe-core tags that make up each level. */
|
|
2
|
+
export const AXE_TAGS = {
|
|
3
|
+
wcag2aa: ["wcag2a", "wcag2aa"],
|
|
4
|
+
wcag21aa: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"],
|
|
5
|
+
wcag22aa: ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa"],
|
|
6
|
+
};
|
|
7
|
+
export const SIZE_PROPS = [
|
|
8
|
+
"font-size",
|
|
9
|
+
"padding",
|
|
10
|
+
"border-radius",
|
|
11
|
+
"gap",
|
|
12
|
+
];
|
|
13
|
+
export const COLOR_PROPS = [
|
|
14
|
+
"color",
|
|
15
|
+
"background-color",
|
|
16
|
+
"border-color",
|
|
17
|
+
];
|
|
18
|
+
/** The line a finding prints, as `factory site compare` gives it. */
|
|
19
|
+
export function formatFinding(f) {
|
|
20
|
+
const at = `${f.page} ${f.width} ${f.scheme}`;
|
|
21
|
+
if (f.kind === "accessibility")
|
|
22
|
+
return `measured: ${at}: accessibility ${f.rule}: ${f.element}: ${f.words}`;
|
|
23
|
+
return `measured: ${at}: ${f.kind}: ${f.words}`;
|
|
24
|
+
}
|
|
25
|
+
// ---- accessibility ----
|
|
26
|
+
/** The failures `after` has that `before` lacks: a failure is the base's when
|
|
27
|
+
* it is the same rule on an element with the same selector or the same
|
|
28
|
+
* markup. `before` is null for a page the base does not have. */
|
|
29
|
+
export function newViolations(before, after, view) {
|
|
30
|
+
const known = new Map();
|
|
31
|
+
for (const v of before ?? []) {
|
|
32
|
+
const k = known.get(v.id) ?? { selectors: new Set(), html: new Set() };
|
|
33
|
+
for (const n of v.nodes) {
|
|
34
|
+
k.selectors.add(n.selector);
|
|
35
|
+
k.html.add(n.html);
|
|
36
|
+
}
|
|
37
|
+
known.set(v.id, k);
|
|
38
|
+
}
|
|
39
|
+
const out = [];
|
|
40
|
+
for (const v of after)
|
|
41
|
+
for (const n of v.nodes) {
|
|
42
|
+
const k = known.get(v.id);
|
|
43
|
+
if (k && (k.selectors.has(n.selector) || k.html.has(n.html)))
|
|
44
|
+
continue;
|
|
45
|
+
out.push({
|
|
46
|
+
...view,
|
|
47
|
+
kind: "accessibility",
|
|
48
|
+
rule: v.id,
|
|
49
|
+
element: n.selector,
|
|
50
|
+
words: n.words,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
// ---- sideways scroll ----
|
|
56
|
+
/** One finding when the branch's page is wider than its window and the base's
|
|
57
|
+
* is not. `before` is the base's scroll width, or null for a missing page. */
|
|
58
|
+
export function sidewaysScroll(before, after, view) {
|
|
59
|
+
if (after <= view.width)
|
|
60
|
+
return [];
|
|
61
|
+
if (before !== null && before > view.width)
|
|
62
|
+
return [];
|
|
63
|
+
return [
|
|
64
|
+
{
|
|
65
|
+
...view,
|
|
66
|
+
kind: "sideways scroll",
|
|
67
|
+
rule: null,
|
|
68
|
+
element: "page",
|
|
69
|
+
words: `the page is ${after} px wide in a ${view.width} px window`,
|
|
70
|
+
},
|
|
71
|
+
];
|
|
72
|
+
}
|
|
73
|
+
export const emptyCensus = () => ({ light: {}, dark: {} });
|
|
74
|
+
export function addToCensus(c, scheme, entries) {
|
|
75
|
+
for (const e of entries)
|
|
76
|
+
(c[scheme][e.prop] ??= new Set()).add(e.value);
|
|
77
|
+
}
|
|
78
|
+
const sizeOf = (v) => Number.parseFloat(v);
|
|
79
|
+
function list(items, unit) {
|
|
80
|
+
const n = items.length;
|
|
81
|
+
if (n === 0)
|
|
82
|
+
return "";
|
|
83
|
+
const joined = n <= 2
|
|
84
|
+
? items.join(" and ")
|
|
85
|
+
: `${items.slice(0, -1).join(", ")}, and ${items[n - 1]}`;
|
|
86
|
+
return `${joined} ${unit}`.trim();
|
|
87
|
+
}
|
|
88
|
+
/** The sizes the site uses for a property: all of them, or the 12 nearest
|
|
89
|
+
* to `near` when there are more, in ascending order. */
|
|
90
|
+
export function nearestSizes(used, near) {
|
|
91
|
+
const sizes = [...used].filter((v) => v.endsWith("px")).map(sizeOf);
|
|
92
|
+
const picked = sizes.length > 12
|
|
93
|
+
? [...sizes]
|
|
94
|
+
.sort((a, b) => Math.abs(a - near) - Math.abs(b - near) || a - b)
|
|
95
|
+
.slice(0, 12)
|
|
96
|
+
: sizes;
|
|
97
|
+
return picked.sort((a, b) => a - b).map(String);
|
|
98
|
+
}
|
|
99
|
+
/** One finding for each value of a property on a changed view of the branch
|
|
100
|
+
* that the base uses in no view of that scheme. */
|
|
101
|
+
export function newValues(base, after, view) {
|
|
102
|
+
const out = [];
|
|
103
|
+
for (const e of after) {
|
|
104
|
+
const used = base[view.scheme][e.prop] ?? new Set();
|
|
105
|
+
if (used.has(e.value))
|
|
106
|
+
continue;
|
|
107
|
+
const on = `${e.tag}${e.text ? ` "${e.text}"` : ""}`;
|
|
108
|
+
const more = e.count > 1 ? ` and ${e.count - 1} more` : "";
|
|
109
|
+
const isColor = COLOR_PROPS.includes(e.prop);
|
|
110
|
+
const sizes = [
|
|
111
|
+
list(nearestSizes(used, sizeOf(e.value)), "px"),
|
|
112
|
+
list([...used].filter((v) => !v.endsWith("px")).sort(), ""),
|
|
113
|
+
].filter(Boolean);
|
|
114
|
+
const tail = isColor
|
|
115
|
+
? `the site uses ${used.size} in ${view.scheme}`
|
|
116
|
+
: sizes.length
|
|
117
|
+
? `the site uses ${sizes.join(", and ")}`
|
|
118
|
+
: "the site uses none";
|
|
119
|
+
out.push({
|
|
120
|
+
...view,
|
|
121
|
+
kind: "new value",
|
|
122
|
+
rule: null,
|
|
123
|
+
element: on,
|
|
124
|
+
words: `${e.prop} ${e.value} on ${on}${more}; ${tail}`,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
return out;
|
|
128
|
+
}
|
|
129
|
+
// ---- collecting, in the page ----
|
|
130
|
+
/** A size as the census keeps it, or null to leave it out: pixels rounded to
|
|
131
|
+
* a tenth, and a percentage, which a radius can compute to, as written. */
|
|
132
|
+
export function sizeValue(raw) {
|
|
133
|
+
const n = Math.round(Number.parseFloat(raw) * 10) / 10;
|
|
134
|
+
if (!Number.isFinite(n) || n <= 0)
|
|
135
|
+
return null;
|
|
136
|
+
return raw.trim().endsWith("%") ? raw.trim() : `${n}px`;
|
|
137
|
+
}
|
|
138
|
+
/** Whether a computed color is fully transparent. Chrome keeps a color's
|
|
139
|
+
* own notation, so the alpha is read after a comma, as in `rgba(0, 0, 0, 0)`,
|
|
140
|
+
* and after a slash, as in `oklab(0 0 0 / 0)`. */
|
|
141
|
+
export function isClear(color) {
|
|
142
|
+
if (!color || color === "transparent")
|
|
143
|
+
return true;
|
|
144
|
+
const alpha = /\/\s*([\d.]+)%?\s*\)$/.exec(color) ??
|
|
145
|
+
/^(?:rgba|hsla)\(.*,\s*([\d.]+)%?\s*\)$/.exec(color);
|
|
146
|
+
return alpha !== null && Number(alpha[1]) === 0;
|
|
147
|
+
}
|
|
148
|
+
/** The entries as the census keeps them, from what the page reported: a size
|
|
149
|
+
* rounded, a fully transparent color left out, and entries that come to the
|
|
150
|
+
* same value joined, the first naming the element. */
|
|
151
|
+
export function tidy(raw) {
|
|
152
|
+
const kept = new Map();
|
|
153
|
+
for (const e of raw) {
|
|
154
|
+
const isColor = COLOR_PROPS.includes(e.prop);
|
|
155
|
+
const value = isColor
|
|
156
|
+
? isClear(e.value)
|
|
157
|
+
? null
|
|
158
|
+
: e.value
|
|
159
|
+
: sizeValue(e.value);
|
|
160
|
+
if (!value)
|
|
161
|
+
continue;
|
|
162
|
+
const hit = kept.get(`${e.prop}|${value}`);
|
|
163
|
+
if (hit)
|
|
164
|
+
hit.count += e.count;
|
|
165
|
+
else
|
|
166
|
+
kept.set(`${e.prop}|${value}`, { ...e, value });
|
|
167
|
+
}
|
|
168
|
+
return [...kept.values()];
|
|
169
|
+
}
|
|
170
|
+
/** Run in the page: every visible element's values as the browser computed
|
|
171
|
+
* them, which `tidy` then rounds and filters. A border's color is reported
|
|
172
|
+
* only where the border has a width. A padding is its four sides, a radius
|
|
173
|
+
* its four corners, and a gap its row and column. */
|
|
174
|
+
export const COLLECT = `(() => {
|
|
175
|
+
const found = new Map();
|
|
176
|
+
const add = (prop, value, el) => {
|
|
177
|
+
if (!value) return;
|
|
178
|
+
const key = prop + '|' + value;
|
|
179
|
+
const hit = found.get(key);
|
|
180
|
+
if (hit) { hit.count++; return; }
|
|
181
|
+
const own = [...el.childNodes].filter((n) => n.nodeType === 3)
|
|
182
|
+
.map((n) => n.textContent).join(' ').replace(/\\s+/g, ' ').trim();
|
|
183
|
+
const text = (own || (el.textContent || '').replace(/\\s+/g, ' ').trim()).slice(0, 40);
|
|
184
|
+
found.set(key, { prop, value, tag: el.tagName.toLowerCase(), text, count: 1 });
|
|
185
|
+
};
|
|
186
|
+
for (const el of document.body.querySelectorAll('*')) {
|
|
187
|
+
if (['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'HEAD'].includes(el.tagName)) continue;
|
|
188
|
+
const r = el.getBoundingClientRect();
|
|
189
|
+
const s = getComputedStyle(el);
|
|
190
|
+
if (!r.width || !r.height || s.display === 'none' || s.visibility === 'hidden') continue;
|
|
191
|
+
const hasText = [...el.childNodes].some((n) => n.nodeType === 3 && n.textContent.trim());
|
|
192
|
+
if (hasText) {
|
|
193
|
+
add('font-size', s.fontSize, el);
|
|
194
|
+
add('color', s.color, el);
|
|
195
|
+
}
|
|
196
|
+
add('background-color', s.backgroundColor, el);
|
|
197
|
+
for (const side of ['Top', 'Right', 'Bottom', 'Left']) {
|
|
198
|
+
if (parseFloat(s['border' + side + 'Width']) > 0)
|
|
199
|
+
add('border-color', s['border' + side + 'Color'], el);
|
|
200
|
+
add('padding', s['padding' + side], el);
|
|
201
|
+
}
|
|
202
|
+
for (const c of ['TopLeft', 'TopRight', 'BottomRight', 'BottomLeft'])
|
|
203
|
+
add('border-radius', s['border' + c + 'Radius'].split(' ')[0], el);
|
|
204
|
+
add('gap', s.rowGap, el);
|
|
205
|
+
add('gap', s.columnGap, el);
|
|
206
|
+
}
|
|
207
|
+
return { scrollWidth: document.documentElement.scrollWidth, entries: [...found.values()] };
|
|
208
|
+
})()`;
|
|
209
|
+
/** Run in the page after axe-core's source: the failures of the given tags,
|
|
210
|
+
* outside frames. */
|
|
211
|
+
export const axeRun = (tags) => `(async () => {
|
|
212
|
+
const res = await axe.run(document, {
|
|
213
|
+
runOnly: { type: 'tag', values: ${JSON.stringify(tags)} },
|
|
214
|
+
iframes: false,
|
|
215
|
+
resultTypes: ['violations'],
|
|
216
|
+
});
|
|
217
|
+
return res.violations.map((v) => ({
|
|
218
|
+
id: v.id,
|
|
219
|
+
nodes: v.nodes.map((n) => ({
|
|
220
|
+
selector: n.target.join(' '),
|
|
221
|
+
html: n.html,
|
|
222
|
+
words: [...n.any, ...n.all, ...n.none].map((c) => c.message).join('; '),
|
|
223
|
+
})),
|
|
224
|
+
}));
|
|
225
|
+
})()`;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** The path of each `<loc>` in a sitemap, whatever host it names. */
|
|
2
|
+
export declare function sitemapPaths(xml: string): string[];
|
|
3
|
+
/** A path as one page's name: without a query or fragment. */
|
|
4
|
+
export declare function normalize(path: string): string;
|
|
5
|
+
/** The paths on the same site that a page's links lead to. `pagePath` is the
|
|
6
|
+
* page the links are on, for relative ones. */
|
|
7
|
+
export declare function linkPaths(html: string, pagePath?: string): string[];
|
|
8
|
+
/** Breadth-first from the home page, `depth` levels, at most `limit` pages.
|
|
9
|
+
* `fetchHtml` returns a page's HTML, or null when it cannot be read. */
|
|
10
|
+
export declare function crawl(fetchHtml: (path: string) => Promise<string | null>, depth?: number, limit?: number): Promise<string[]>;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// Finding the pages of a website: from its sitemap, or from the links on its
|
|
2
|
+
// home page. Both give paths only, since a sitemap names the live site and the
|
|
3
|
+
// pages wanted are the same paths on the two sites being compared.
|
|
4
|
+
const SKIP_EXTENSION = /\.(?:png|jpe?g|gif|svg|webp|avif|ico|pdf|zip|css|js|mjs|json|xml|txt|mp4|webm|mp3|woff2?|ttf)$/i;
|
|
5
|
+
/** The path of each `<loc>` in a sitemap, whatever host it names. */
|
|
6
|
+
export function sitemapPaths(xml) {
|
|
7
|
+
const paths = [];
|
|
8
|
+
for (const m of xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g)) {
|
|
9
|
+
try {
|
|
10
|
+
paths.push(normalize(new URL(m[1].replaceAll("&", "&")).pathname));
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
// Not an address: not a page.
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
return [...new Set(paths)];
|
|
17
|
+
}
|
|
18
|
+
/** A path as one page's name: without a query or fragment. */
|
|
19
|
+
export function normalize(path) {
|
|
20
|
+
const p = path.split(/[?#]/)[0] || "/";
|
|
21
|
+
return p.startsWith("/") ? p : `/${p}`;
|
|
22
|
+
}
|
|
23
|
+
/** The paths on the same site that a page's links lead to. `pagePath` is the
|
|
24
|
+
* page the links are on, for relative ones. */
|
|
25
|
+
export function linkPaths(html, pagePath = "/") {
|
|
26
|
+
const origin = "http://site.invalid";
|
|
27
|
+
const found = [];
|
|
28
|
+
for (const m of html.matchAll(/<a\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)')/gi)) {
|
|
29
|
+
const href = (m[1] ?? m[2] ?? "").trim();
|
|
30
|
+
if (!href || /^(?:mailto|tel|javascript|data):/i.test(href))
|
|
31
|
+
continue;
|
|
32
|
+
try {
|
|
33
|
+
const url = new URL(href, `${origin}${pagePath}`);
|
|
34
|
+
if (url.origin !== origin && !/^\/\/|^[a-z]+:/i.test(href))
|
|
35
|
+
continue;
|
|
36
|
+
if (url.origin !== origin)
|
|
37
|
+
continue;
|
|
38
|
+
if (SKIP_EXTENSION.test(url.pathname))
|
|
39
|
+
continue;
|
|
40
|
+
found.push(normalize(url.pathname));
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// Not a link.
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return [...new Set(found)];
|
|
47
|
+
}
|
|
48
|
+
/** Breadth-first from the home page, `depth` levels, at most `limit` pages.
|
|
49
|
+
* `fetchHtml` returns a page's HTML, or null when it cannot be read. */
|
|
50
|
+
export async function crawl(fetchHtml, depth = 2, limit = 50) {
|
|
51
|
+
const seen = ["/"];
|
|
52
|
+
let level = ["/"];
|
|
53
|
+
for (let d = 0; d < depth && seen.length < limit; d++) {
|
|
54
|
+
const next = [];
|
|
55
|
+
for (const path of level) {
|
|
56
|
+
const html = await fetchHtml(path);
|
|
57
|
+
if (html === null)
|
|
58
|
+
continue;
|
|
59
|
+
for (const link of linkPaths(html, path)) {
|
|
60
|
+
if (seen.length >= limit)
|
|
61
|
+
break;
|
|
62
|
+
if (!seen.includes(link)) {
|
|
63
|
+
seen.push(link);
|
|
64
|
+
next.push(link);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
level = next;
|
|
69
|
+
}
|
|
70
|
+
return seen;
|
|
71
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/** A decoded picture: RGBA, four bytes a pixel. */
|
|
2
|
+
export type Picture = {
|
|
3
|
+
width: number;
|
|
4
|
+
height: number;
|
|
5
|
+
data: Uint8Array;
|
|
6
|
+
};
|
|
7
|
+
/** A color channel has to differ by more than this, of 255, to count. */
|
|
8
|
+
export declare const TOLERANCE = 8;
|
|
9
|
+
export declare const STRIP = 4000;
|
|
10
|
+
/** The strips a page of `height` px is read in. */
|
|
11
|
+
export declare function strips(height: number, size?: number): {
|
|
12
|
+
top: number;
|
|
13
|
+
height: number;
|
|
14
|
+
}[];
|
|
15
|
+
export type Diff = {
|
|
16
|
+
width: number;
|
|
17
|
+
height: number;
|
|
18
|
+
/** One byte a pixel, 1 where the pictures differ. */
|
|
19
|
+
mask: Uint8Array;
|
|
20
|
+
count: number;
|
|
21
|
+
/** The first and last row with a difference, or null. */
|
|
22
|
+
rows: [number, number] | null;
|
|
23
|
+
};
|
|
24
|
+
/** The pixels where either picture is missing, or a channel differs by more
|
|
25
|
+
* than the tolerance. The pictures may be different sizes: what only one has
|
|
26
|
+
* differs. Read strip by strip, so a tall page is walked in bounded pieces. */
|
|
27
|
+
export declare function diff(a: Picture, b: Picture, tolerance?: number): Diff;
|
|
28
|
+
/** The mask grown by `by` px on every side, so noise that moves a little
|
|
29
|
+
* between two shots still covers where it was. */
|
|
30
|
+
export declare function grow(d: Diff, byX: number, byY?: number): Uint8Array;
|
|
31
|
+
/** Each connected patch of `mask` filled out to its bounding box, so a clock
|
|
32
|
+
* or a carousel is covered whole, wherever its pixels happened to differ. */
|
|
33
|
+
export declare function boxes(mask: Uint8Array, width: number, height: number): Uint8Array<ArrayBuffer>;
|
|
34
|
+
/** What changed between `before` and `after`, leaving out whatever differs
|
|
35
|
+
* between two pictures of one site: that is noise, a clock or a carousel, and
|
|
36
|
+
* needs no setting. `again` is a second picture of the after site. With
|
|
37
|
+
* `beforeAgain`, a second picture of the before site, a pixel must also differ
|
|
38
|
+
* between the two second pictures, so a page caught half-loaded in one shot
|
|
39
|
+
* is not reported as changed. */
|
|
40
|
+
export declare function changedWithoutNoise(before: Picture, after: Picture, again: Picture, beforeAgain?: Picture, noiseMargin?: number, noiseMarginX?: number): Diff;
|
|
41
|
+
export type Status = "same" | "changed" | "new" | "removed";
|
|
42
|
+
/** A page and view: new when only the after site has the page, removed when
|
|
43
|
+
* only the before site does, changed when a pixel differs. */
|
|
44
|
+
export declare function classify(has: {
|
|
45
|
+
before: boolean;
|
|
46
|
+
after: boolean;
|
|
47
|
+
}, changedPixels: number): Status;
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// Comparing two pictures of a page: which pixels changed, which of those are
|
|
2
|
+
// noise, and what to call a page and view. No browser: these take decoded
|
|
3
|
+
// pictures, so each is a plain function.
|
|
4
|
+
/** A color channel has to differ by more than this, of 255, to count. */
|
|
5
|
+
export const TOLERANCE = 8;
|
|
6
|
+
export const STRIP = 4000;
|
|
7
|
+
/** The strips a page of `height` px is read in. */
|
|
8
|
+
export function strips(height, size = STRIP) {
|
|
9
|
+
const out = [];
|
|
10
|
+
for (let top = 0; top < height; top += size)
|
|
11
|
+
out.push({ top, height: Math.min(size, height - top) });
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
/** The pixels where either picture is missing, or a channel differs by more
|
|
15
|
+
* than the tolerance. The pictures may be different sizes: what only one has
|
|
16
|
+
* differs. Read strip by strip, so a tall page is walked in bounded pieces. */
|
|
17
|
+
export function diff(a, b, tolerance = TOLERANCE) {
|
|
18
|
+
const width = Math.max(a.width, b.width);
|
|
19
|
+
const height = Math.max(a.height, b.height);
|
|
20
|
+
const mask = new Uint8Array(width * height);
|
|
21
|
+
let count = 0;
|
|
22
|
+
let first = -1;
|
|
23
|
+
let last = -1;
|
|
24
|
+
for (const strip of strips(height)) {
|
|
25
|
+
for (let y = strip.top; y < strip.top + strip.height; y++) {
|
|
26
|
+
for (let x = 0; x < width; x++) {
|
|
27
|
+
const inA = x < a.width && y < a.height;
|
|
28
|
+
const inB = x < b.width && y < b.height;
|
|
29
|
+
let changed = inA !== inB;
|
|
30
|
+
if (inA && inB) {
|
|
31
|
+
const i = (y * a.width + x) * 4;
|
|
32
|
+
const j = (y * b.width + x) * 4;
|
|
33
|
+
for (let c = 0; c < 4 && !changed; c++)
|
|
34
|
+
changed = Math.abs(a.data[i + c] - b.data[j + c]) > tolerance;
|
|
35
|
+
}
|
|
36
|
+
if (changed) {
|
|
37
|
+
mask[y * width + x] = 1;
|
|
38
|
+
count++;
|
|
39
|
+
if (first < 0)
|
|
40
|
+
first = y;
|
|
41
|
+
last = y;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return { width, height, mask, count, rows: first < 0 ? null : [first, last] };
|
|
47
|
+
}
|
|
48
|
+
/** The mask grown by `by` px on every side, so noise that moves a little
|
|
49
|
+
* between two shots still covers where it was. */
|
|
50
|
+
export function grow(d, byX, byY = byX) {
|
|
51
|
+
const out = new Uint8Array(d.mask.length);
|
|
52
|
+
for (let y = 0; y < d.height; y++)
|
|
53
|
+
for (let x = 0; x < d.width; x++) {
|
|
54
|
+
if (!d.mask[y * d.width + x])
|
|
55
|
+
continue;
|
|
56
|
+
for (let dy = -byY; dy <= byY; dy++)
|
|
57
|
+
for (let dx = -byX; dx <= byX; dx++) {
|
|
58
|
+
const yy = y + dy;
|
|
59
|
+
const xx = x + dx;
|
|
60
|
+
if (yy >= 0 && yy < d.height && xx >= 0 && xx < d.width)
|
|
61
|
+
out[yy * d.width + xx] = 1;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
/** Each connected patch of `mask` filled out to its bounding box, so a clock
|
|
67
|
+
* or a carousel is covered whole, wherever its pixels happened to differ. */
|
|
68
|
+
export function boxes(mask, width, height) {
|
|
69
|
+
const out = new Uint8Array(mask.length);
|
|
70
|
+
const seen = new Uint8Array(mask.length);
|
|
71
|
+
for (let start = 0; start < mask.length; start++) {
|
|
72
|
+
if (!mask[start] || seen[start])
|
|
73
|
+
continue;
|
|
74
|
+
let [x0, y0, x1, y1] = [width, height, 0, 0];
|
|
75
|
+
const stack = [start];
|
|
76
|
+
seen[start] = 1;
|
|
77
|
+
while (stack.length) {
|
|
78
|
+
const at = stack.pop();
|
|
79
|
+
const x = at % width;
|
|
80
|
+
const y = (at - x) / width;
|
|
81
|
+
x0 = Math.min(x0, x);
|
|
82
|
+
x1 = Math.max(x1, x);
|
|
83
|
+
y0 = Math.min(y0, y);
|
|
84
|
+
y1 = Math.max(y1, y);
|
|
85
|
+
for (const [nx, ny] of [
|
|
86
|
+
[x - 1, y],
|
|
87
|
+
[x + 1, y],
|
|
88
|
+
[x, y - 1],
|
|
89
|
+
[x, y + 1],
|
|
90
|
+
]) {
|
|
91
|
+
if (nx < 0 || ny < 0 || nx >= width || ny >= height)
|
|
92
|
+
continue;
|
|
93
|
+
const next = ny * width + nx;
|
|
94
|
+
if (mask[next] && !seen[next]) {
|
|
95
|
+
seen[next] = 1;
|
|
96
|
+
stack.push(next);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
for (let y = y0; y <= y1; y++)
|
|
101
|
+
out.fill(1, y * width + x0, y * width + x1 + 1);
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
const at = (m, w, h, x, y) => x < w && y < h ? m[y * w + x] : 0;
|
|
106
|
+
/** What changed between `before` and `after`, leaving out whatever differs
|
|
107
|
+
* between two pictures of one site: that is noise, a clock or a carousel, and
|
|
108
|
+
* needs no setting. `again` is a second picture of the after site. With
|
|
109
|
+
* `beforeAgain`, a second picture of the before site, a pixel must also differ
|
|
110
|
+
* between the two second pictures, so a page caught half-loaded in one shot
|
|
111
|
+
* is not reported as changed. */
|
|
112
|
+
export function changedWithoutNoise(before, after, again, beforeAgain, noiseMargin = 12, noiseMarginX = 60) {
|
|
113
|
+
const first = diff(before, after);
|
|
114
|
+
const second = beforeAgain ? diff(beforeAgain, again) : null;
|
|
115
|
+
const noises = [
|
|
116
|
+
diff(after, again),
|
|
117
|
+
...(beforeAgain ? [diff(before, beforeAgain)] : []),
|
|
118
|
+
].map((d) => ({
|
|
119
|
+
d,
|
|
120
|
+
mask: boxes(grow(d, noiseMarginX, noiseMargin), d.width, d.height),
|
|
121
|
+
}));
|
|
122
|
+
const { width, height } = first;
|
|
123
|
+
const mask = new Uint8Array(first.mask.length);
|
|
124
|
+
let count = 0;
|
|
125
|
+
let firstRow = -1;
|
|
126
|
+
let lastRow = -1;
|
|
127
|
+
for (let y = 0; y < height; y++)
|
|
128
|
+
for (let x = 0; x < width; x++) {
|
|
129
|
+
if (!first.mask[y * width + x])
|
|
130
|
+
continue;
|
|
131
|
+
if (second && !at(second.mask, second.width, second.height, x, y))
|
|
132
|
+
continue;
|
|
133
|
+
if (noises.some((n) => at(n.mask, n.d.width, n.d.height, x, y)))
|
|
134
|
+
continue;
|
|
135
|
+
mask[y * width + x] = 1;
|
|
136
|
+
count++;
|
|
137
|
+
if (firstRow < 0)
|
|
138
|
+
firstRow = y;
|
|
139
|
+
lastRow = y;
|
|
140
|
+
}
|
|
141
|
+
return {
|
|
142
|
+
width,
|
|
143
|
+
height,
|
|
144
|
+
mask,
|
|
145
|
+
count,
|
|
146
|
+
rows: firstRow < 0 ? null : [firstRow, lastRow],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
/** A page and view: new when only the after site has the page, removed when
|
|
150
|
+
* only the before site does, changed when a pixel differs. */
|
|
151
|
+
export function classify(has, changedPixels) {
|
|
152
|
+
if (has.after && !has.before)
|
|
153
|
+
return "new";
|
|
154
|
+
if (has.before && !has.after)
|
|
155
|
+
return "removed";
|
|
156
|
+
return changedPixels > 0 ? "changed" : "same";
|
|
157
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
export declare class SiteError extends Error {
|
|
2
|
+
}
|
|
3
|
+
export type Running = {
|
|
4
|
+
url: string;
|
|
5
|
+
pid: number;
|
|
6
|
+
/** The last lines the command printed. */
|
|
7
|
+
tail: () => string[];
|
|
8
|
+
};
|
|
9
|
+
/** A port nothing is listening on. */
|
|
10
|
+
export declare function freePort(): Promise<number>;
|
|
11
|
+
/** `url` with `$PORT` filled in. */
|
|
12
|
+
export declare const withPort: (text: string, port: number) => string;
|
|
13
|
+
/** Run `start` from `cwd` with PORT set, and wait up to `minutes` for `url`
|
|
14
|
+
* to answer. Fails with the last lines the command printed. */
|
|
15
|
+
export declare function startSite(opts: {
|
|
16
|
+
start: string;
|
|
17
|
+
url: string;
|
|
18
|
+
cwd: string;
|
|
19
|
+
minutes: number;
|
|
20
|
+
port: number;
|
|
21
|
+
}, onSpawn?: (pid: number) => void): Promise<Running>;
|
|
22
|
+
/** Starts every site at once, and resolves when each answers. Each process
|
|
23
|
+
* is in `pids` from the moment it is spawned: when one site fails to start,
|
|
24
|
+
* or the run is interrupted while they start, the caller's cleanup is all
|
|
25
|
+
* that stops the others. */
|
|
26
|
+
export declare function startSites(sites: Parameters<typeof startSite>[0][], pids: number[]): Promise<Running[]>;
|
|
27
|
+
export declare function stopProcess(pid: number): void;
|
|
28
|
+
/** Whether the repository is a git checkout with this ref. */
|
|
29
|
+
export declare const defaultBranch: (root: string) => string;
|
|
30
|
+
/** A detached worktree of `ref` in a temporary folder, with its dependencies
|
|
31
|
+
* installed when it has a lockfile. Returns its path. */
|
|
32
|
+
export declare function addWorktree(root: string, ref: string): string;
|
|
33
|
+
export declare function removeWorktree(root: string, dir: string): void;
|
|
34
|
+
export type Kept = {
|
|
35
|
+
before: {
|
|
36
|
+
url: string;
|
|
37
|
+
pid: number;
|
|
38
|
+
};
|
|
39
|
+
after: {
|
|
40
|
+
url: string;
|
|
41
|
+
pid: number;
|
|
42
|
+
};
|
|
43
|
+
worktree: string;
|
|
44
|
+
};
|
|
45
|
+
export declare function keep(root: string, kept: Kept): void;
|
|
46
|
+
/** Ends what `--keep` left running and removes its worktree. */
|
|
47
|
+
export declare function stopKept(root: string): boolean;
|