dembrandt 0.32.0 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/lib/robots.d.ts +5 -0
- package/dist/lib/robots.js +6 -1
- package/dist/lib/robots.js.map +1 -1
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -62,7 +62,7 @@ Default is formatted terminal output only. Full flag reference in **[docs/usage.
|
|
|
62
62
|
Extract a preview deployment, compare against a committed baseline, fail the job when tokens moved:
|
|
63
63
|
|
|
64
64
|
```yaml
|
|
65
|
-
- uses: dembrandt/dembrandt@v0.32.
|
|
65
|
+
- uses: dembrandt/dembrandt@v0.32.1
|
|
66
66
|
with:
|
|
67
67
|
url: https://preview.example.com
|
|
68
68
|
baseline: .dembrandt/baseline.json
|
package/dist/lib/robots.d.ts
CHANGED
|
@@ -5,6 +5,9 @@ interface RobotsRule {
|
|
|
5
5
|
value: string;
|
|
6
6
|
}
|
|
7
7
|
export type RobotsResult = {
|
|
8
|
+
status: "absent";
|
|
9
|
+
robotsUrl: string;
|
|
10
|
+
} | {
|
|
8
11
|
status: "unavailable";
|
|
9
12
|
robotsUrl: string;
|
|
10
13
|
} | {
|
|
@@ -14,6 +17,8 @@ export type RobotsResult = {
|
|
|
14
17
|
rule: string | null;
|
|
15
18
|
};
|
|
16
19
|
export type RobotsRules = {
|
|
20
|
+
status: "absent";
|
|
21
|
+
} | {
|
|
17
22
|
status: "unavailable";
|
|
18
23
|
} | {
|
|
19
24
|
status: "ok";
|
package/dist/lib/robots.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/** The name a site addresses us by. Only applies when we send it (see `agent`). */
|
|
2
2
|
export const ROBOTS_AGENT = "Dembrandt";
|
|
3
|
+
const ABSENT_STATUSES = new Set([404, 410]);
|
|
3
4
|
/**
|
|
4
5
|
* Fetch and parse robots.txt for the target's origin once, so a multi-page
|
|
5
6
|
* crawl can check every discovered URL against it without one request per page.
|
|
@@ -15,6 +16,8 @@ export async function fetchRobotsRules(targetUrl, { timeoutMs = 5000, agent = "*
|
|
|
15
16
|
signal: controller.signal,
|
|
16
17
|
headers: { "User-Agent": ROBOTS_AGENT },
|
|
17
18
|
});
|
|
19
|
+
if (ABSENT_STATUSES.has(res.status))
|
|
20
|
+
return { status: "absent" };
|
|
18
21
|
if (!res.ok)
|
|
19
22
|
return { status: "unavailable" };
|
|
20
23
|
body = await res.text();
|
|
@@ -39,6 +42,8 @@ export async function fetchRobotsRules(targetUrl, { timeoutMs = 5000, agent = "*
|
|
|
39
42
|
* because it is the user, not the tool, who knows what they may fetch.
|
|
40
43
|
*/
|
|
41
44
|
export function robotsVerdict(robots, { enforce }) {
|
|
45
|
+
if (robots.status === 'absent')
|
|
46
|
+
return { action: 'proceed' };
|
|
42
47
|
if (robots.status === 'unavailable') {
|
|
43
48
|
return enforce ? { action: 'refuse', reason: 'robots.txt could not be read' } : { action: 'proceed' };
|
|
44
49
|
}
|
|
@@ -54,7 +59,7 @@ export function evaluatePath(rules, path) {
|
|
|
54
59
|
export function checkAgainstRules(targetUrl, rules) {
|
|
55
60
|
const u = new URL(targetUrl);
|
|
56
61
|
if (rules.status !== "ok") {
|
|
57
|
-
return { status:
|
|
62
|
+
return { status: rules.status, robotsUrl: `${u.protocol}//${u.host}/robots.txt` };
|
|
58
63
|
}
|
|
59
64
|
return { status: "ok", robotsUrl: rules.robotsUrl, ...evaluatePath(rules.rules, u.pathname || "/") };
|
|
60
65
|
}
|
package/dist/lib/robots.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../lib/robots.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../lib/robots.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,MAAM,CAAC,MAAM,YAAY,GAAG,WAAW,CAAC;AAYxC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAY5C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAiB,EACjB,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,GAAG,GAAG,KAA6C,EAAE;IAE9E,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC;IAExD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;YACjC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,EAAE,YAAY,EAAE,YAAY,EAAE;SACxC,CAAC,CAAC;QACH,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QACjE,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;QAC9C,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,4EAA4E;QAC5E,sEAAsE;QACtE,IAAI,aAAa,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC;IACnC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;AACpF,CAAC;AAOD;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAoB,EACpB,EAAE,OAAO,EAAwB;IAEjC,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,aAAa,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,8BAA8B,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IACxG,CAAC;IACD,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;IAEjD,MAAM,MAAM,GAAG,0CAA0C,MAAM,CAAC,IAAI,IAAI,CAAC;IACzE,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAmB,EAAE,IAAY;IAC5D,OAAO,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,KAAkB;IACrE,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,aAAa,EAAE,CAAC;IACpF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC;AACvG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAAiB,EACjB,OAA+C,EAAE;IAEjD,OAAO,iBAAiB,CAAC,SAAS,EAAE,MAAM,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;AAC/E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAc,EACd,WAAwB;IAExB,IAAI,WAAW,CAAC,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IAC1E,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,UAAU,GAA2C,EAAE,CAAC;IAC9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,QAAQ,CAAC,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACnC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,OAAO,GAAuB,IAAI,CAAC;IACvC,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,SAAS;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEzC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC9B,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACrD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;YACD,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3C,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,0EAA0E;AAC1E,SAAS,sBAAsB,CAAC,IAAY;IAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,CAAC;SACjD,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,yCAAyC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,UAAU,CAAC,MAAqB,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,CAAC,KAAK,CAAC;IAChD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,KAAmB,EAAE,IAAY;IACjD,IAAI,IAAI,GAAyE;QAC/E,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,CAAC,CAAC;QACV,KAAK,EAAE,EAAE;KACV,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC,CAAC,KAAK;YAAE,SAAS;QACvB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;YAAE,SAAS;QAC1C,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YAAE,IAAI,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5E,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;IAC1E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,OAAe;IAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YACxC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YACnC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;YAC/B,CAAC,GAAG,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC;QACzB,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/package.json
CHANGED
package/package.json
CHANGED