@robinpath/robots 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/robots.d.ts +178 -0
- package/dist/robots.d.ts.map +1 -0
- package/dist/robots.js +174 -0
- package/dist/robots.js.map +1 -0
- package/package.json +13 -0
package/README.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# @robinpath/robots
|
|
2
|
+
|
|
3
|
+
> robots.txt parsing, generation, URL permission checking, and crawl configuration
|
|
4
|
+
|
|
5
|
+
   
|
|
6
|
+
|
|
7
|
+
## Why use this module?
|
|
8
|
+
|
|
9
|
+
The `robots` module lets you:
|
|
10
|
+
|
|
11
|
+
- Parse robots.txt content
|
|
12
|
+
- Generate robots.txt
|
|
13
|
+
- Check if URL is allowed
|
|
14
|
+
- Check if URL is disallowed
|
|
15
|
+
- Get crawl delay for agent
|
|
16
|
+
|
|
17
|
+
All functions are callable directly from RobinPath scripts with a simple, consistent API.
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
npm install @robinpath/robots
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Quick Start
|
|
26
|
+
|
|
27
|
+
No credentials needed — start using it right away:
|
|
28
|
+
|
|
29
|
+
```robinpath
|
|
30
|
+
robots.create {"rules": [{"userAgent": "*", "disallow": ["/admin"]}]}
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Available Functions
|
|
34
|
+
|
|
35
|
+
| Function | Description |
|
|
36
|
+
|----------|-------------|
|
|
37
|
+
| `robots.parse` | Parse robots.txt content |
|
|
38
|
+
| `robots.create` | Generate robots.txt |
|
|
39
|
+
| `robots.isAllowed` | Check if URL is allowed |
|
|
40
|
+
| `robots.isDisallowed` | Check if URL is disallowed |
|
|
41
|
+
| `robots.getCrawlDelay` | Get crawl delay for agent |
|
|
42
|
+
| `robots.getSitemaps` | Get sitemap URLs |
|
|
43
|
+
| `robots.getRules` | Get rules for agent |
|
|
44
|
+
| `robots.addRule` | Add rule to robots config |
|
|
45
|
+
| `robots.removeRule` | Remove rule by user agent |
|
|
46
|
+
| `robots.addSitemap` | Add sitemap URL |
|
|
47
|
+
| `robots.allowAll` | Generate allow-all robots.txt |
|
|
48
|
+
| `robots.disallowAll` | Generate disallow-all robots.txt |
|
|
49
|
+
| `robots.fetch` | Fetch and parse robots.txt from URL |
|
|
50
|
+
|
|
51
|
+
## Examples
|
|
52
|
+
|
|
53
|
+
### Generate robots.txt
|
|
54
|
+
|
|
55
|
+
```robinpath
|
|
56
|
+
robots.create {"rules": [{"userAgent": "*", "disallow": ["/admin"]}]}
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Check if URL is allowed
|
|
60
|
+
|
|
61
|
+
```robinpath
|
|
62
|
+
robots.isAllowed $robots "/page" "Googlebot"
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Check if URL is disallowed
|
|
66
|
+
|
|
67
|
+
```robinpath
|
|
68
|
+
robots.isDisallowed $robots "/admin"
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Integration with RobinPath
|
|
72
|
+
|
|
73
|
+
```typescript
|
|
74
|
+
import { RobinPath } from "@wiredwp/robinpath";
|
|
75
|
+
import Module from "@robinpath/robots";
|
|
76
|
+
|
|
77
|
+
const rp = new RobinPath();
|
|
78
|
+
rp.registerModule(Module.name, Module.functions);
|
|
79
|
+
rp.registerModuleMeta(Module.name, Module.functionMetadata);
|
|
80
|
+
|
|
81
|
+
const result = await rp.executeScript(`
|
|
82
|
+
robots.create {"rules": [{"userAgent": "*", "disallow": ["/admin"]}]}
|
|
83
|
+
`);
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Full API Reference
|
|
87
|
+
|
|
88
|
+
See [MODULE.md](./MODULE.md) for complete documentation including all parameters, return types, error handling, and advanced examples.
|
|
89
|
+
|
|
90
|
+
## Related Modules
|
|
91
|
+
|
|
92
|
+
- [`@robinpath/json`](../json) — JSON module for complementary functionality
|
|
93
|
+
|
|
94
|
+
## License
|
|
95
|
+
|
|
96
|
+
MIT
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ModuleAdapter } from "@wiredwp/robinpath";
|
|
2
|
+
declare const RobotsModule: ModuleAdapter;
|
|
3
|
+
export default RobotsModule;
|
|
4
|
+
export { RobotsModule };
|
|
5
|
+
export { RobotsFunctions, RobotsFunctionMetadata, RobotsModuleMetadata } from "./robots.js";
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAExD,QAAA,MAAM,YAAY,EAAE,aAA2K,CAAC;AAChM,eAAe,YAAY,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { RobotsFunctions, RobotsFunctionMetadata, RobotsModuleMetadata } from "./robots.js";
|
|
2
|
+
const RobotsModule = { name: "robots", functions: RobotsFunctions, functionMetadata: RobotsFunctionMetadata, moduleMetadata: RobotsModuleMetadata, global: false };
|
|
3
|
+
export default RobotsModule;
|
|
4
|
+
export { RobotsModule };
|
|
5
|
+
export { RobotsFunctions, RobotsFunctionMetadata, RobotsModuleMetadata } from "./robots.js";
|
|
6
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC5F,MAAM,YAAY,GAAkB,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,eAAe,EAAE,gBAAgB,EAAE,sBAA6B,EAAE,cAAc,EAAE,oBAA2B,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAChM,eAAe,YAAY,CAAC;AAC5B,OAAO,EAAE,YAAY,EAAE,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/robots.d.ts
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import type { BuiltinHandler } from "@wiredwp/robinpath";
|
|
2
|
+
export declare const RobotsFunctions: Record<string, BuiltinHandler>;
|
|
3
|
+
export declare const RobotsFunctionMetadata: {
|
|
4
|
+
parse: {
|
|
5
|
+
description: string;
|
|
6
|
+
parameters: {
|
|
7
|
+
name: string;
|
|
8
|
+
dataType: string;
|
|
9
|
+
description: string;
|
|
10
|
+
formInputType: string;
|
|
11
|
+
required: boolean;
|
|
12
|
+
}[];
|
|
13
|
+
returnType: string;
|
|
14
|
+
returnDescription: string;
|
|
15
|
+
example: string;
|
|
16
|
+
};
|
|
17
|
+
create: {
|
|
18
|
+
description: string;
|
|
19
|
+
parameters: {
|
|
20
|
+
name: string;
|
|
21
|
+
dataType: string;
|
|
22
|
+
description: string;
|
|
23
|
+
formInputType: string;
|
|
24
|
+
required: boolean;
|
|
25
|
+
}[];
|
|
26
|
+
returnType: string;
|
|
27
|
+
returnDescription: string;
|
|
28
|
+
example: string;
|
|
29
|
+
};
|
|
30
|
+
isAllowed: {
|
|
31
|
+
description: string;
|
|
32
|
+
parameters: {
|
|
33
|
+
name: string;
|
|
34
|
+
dataType: string;
|
|
35
|
+
description: string;
|
|
36
|
+
formInputType: string;
|
|
37
|
+
required: boolean;
|
|
38
|
+
}[];
|
|
39
|
+
returnType: string;
|
|
40
|
+
returnDescription: string;
|
|
41
|
+
example: string;
|
|
42
|
+
};
|
|
43
|
+
isDisallowed: {
|
|
44
|
+
description: string;
|
|
45
|
+
parameters: {
|
|
46
|
+
name: string;
|
|
47
|
+
dataType: string;
|
|
48
|
+
description: string;
|
|
49
|
+
formInputType: string;
|
|
50
|
+
required: boolean;
|
|
51
|
+
}[];
|
|
52
|
+
returnType: string;
|
|
53
|
+
returnDescription: string;
|
|
54
|
+
example: string;
|
|
55
|
+
};
|
|
56
|
+
getCrawlDelay: {
|
|
57
|
+
description: string;
|
|
58
|
+
parameters: {
|
|
59
|
+
name: string;
|
|
60
|
+
dataType: string;
|
|
61
|
+
description: string;
|
|
62
|
+
formInputType: string;
|
|
63
|
+
required: boolean;
|
|
64
|
+
}[];
|
|
65
|
+
returnType: string;
|
|
66
|
+
returnDescription: string;
|
|
67
|
+
example: string;
|
|
68
|
+
};
|
|
69
|
+
getSitemaps: {
|
|
70
|
+
description: string;
|
|
71
|
+
parameters: {
|
|
72
|
+
name: string;
|
|
73
|
+
dataType: string;
|
|
74
|
+
description: string;
|
|
75
|
+
formInputType: string;
|
|
76
|
+
required: boolean;
|
|
77
|
+
}[];
|
|
78
|
+
returnType: string;
|
|
79
|
+
returnDescription: string;
|
|
80
|
+
example: string;
|
|
81
|
+
};
|
|
82
|
+
getRules: {
|
|
83
|
+
description: string;
|
|
84
|
+
parameters: {
|
|
85
|
+
name: string;
|
|
86
|
+
dataType: string;
|
|
87
|
+
description: string;
|
|
88
|
+
formInputType: string;
|
|
89
|
+
required: boolean;
|
|
90
|
+
}[];
|
|
91
|
+
returnType: string;
|
|
92
|
+
returnDescription: string;
|
|
93
|
+
example: string;
|
|
94
|
+
};
|
|
95
|
+
addRule: {
|
|
96
|
+
description: string;
|
|
97
|
+
parameters: {
|
|
98
|
+
name: string;
|
|
99
|
+
dataType: string;
|
|
100
|
+
description: string;
|
|
101
|
+
formInputType: string;
|
|
102
|
+
required: boolean;
|
|
103
|
+
}[];
|
|
104
|
+
returnType: string;
|
|
105
|
+
returnDescription: string;
|
|
106
|
+
example: string;
|
|
107
|
+
};
|
|
108
|
+
removeRule: {
|
|
109
|
+
description: string;
|
|
110
|
+
parameters: {
|
|
111
|
+
name: string;
|
|
112
|
+
dataType: string;
|
|
113
|
+
description: string;
|
|
114
|
+
formInputType: string;
|
|
115
|
+
required: boolean;
|
|
116
|
+
}[];
|
|
117
|
+
returnType: string;
|
|
118
|
+
returnDescription: string;
|
|
119
|
+
example: string;
|
|
120
|
+
};
|
|
121
|
+
addSitemap: {
|
|
122
|
+
description: string;
|
|
123
|
+
parameters: {
|
|
124
|
+
name: string;
|
|
125
|
+
dataType: string;
|
|
126
|
+
description: string;
|
|
127
|
+
formInputType: string;
|
|
128
|
+
required: boolean;
|
|
129
|
+
}[];
|
|
130
|
+
returnType: string;
|
|
131
|
+
returnDescription: string;
|
|
132
|
+
example: string;
|
|
133
|
+
};
|
|
134
|
+
allowAll: {
|
|
135
|
+
description: string;
|
|
136
|
+
parameters: {
|
|
137
|
+
name: string;
|
|
138
|
+
dataType: string;
|
|
139
|
+
description: string;
|
|
140
|
+
formInputType: string;
|
|
141
|
+
required: boolean;
|
|
142
|
+
}[];
|
|
143
|
+
returnType: string;
|
|
144
|
+
returnDescription: string;
|
|
145
|
+
example: string;
|
|
146
|
+
};
|
|
147
|
+
disallowAll: {
|
|
148
|
+
description: string;
|
|
149
|
+
parameters: {
|
|
150
|
+
name: string;
|
|
151
|
+
dataType: string;
|
|
152
|
+
description: string;
|
|
153
|
+
formInputType: string;
|
|
154
|
+
required: boolean;
|
|
155
|
+
}[];
|
|
156
|
+
returnType: string;
|
|
157
|
+
returnDescription: string;
|
|
158
|
+
example: string;
|
|
159
|
+
};
|
|
160
|
+
fetch: {
|
|
161
|
+
description: string;
|
|
162
|
+
parameters: {
|
|
163
|
+
name: string;
|
|
164
|
+
dataType: string;
|
|
165
|
+
description: string;
|
|
166
|
+
formInputType: string;
|
|
167
|
+
required: boolean;
|
|
168
|
+
}[];
|
|
169
|
+
returnType: string;
|
|
170
|
+
returnDescription: string;
|
|
171
|
+
example: string;
|
|
172
|
+
};
|
|
173
|
+
};
|
|
174
|
+
export declare const RobotsModuleMetadata: {
|
|
175
|
+
description: string;
|
|
176
|
+
methods: string[];
|
|
177
|
+
};
|
|
178
|
+
//# sourceMappingURL=robots.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAA2C,MAAM,oBAAoB,CAAC;AA2JlG,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAgK,CAAC;AAE5N,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAclC,CAAC;AAEF,eAAO,MAAM,oBAAoB;;;CAGhC,CAAC"}
|
package/dist/robots.js
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
const parse = (args) => {
|
|
2
|
+
const text = String(args[0] ?? "");
|
|
3
|
+
const rules = [];
|
|
4
|
+
const sitemaps = [];
|
|
5
|
+
let host;
|
|
6
|
+
let current = null;
|
|
7
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
8
|
+
const line = rawLine.replace(/#.*$/, "").trim();
|
|
9
|
+
if (!line)
|
|
10
|
+
continue;
|
|
11
|
+
const colonIdx = line.indexOf(":");
|
|
12
|
+
if (colonIdx < 0)
|
|
13
|
+
continue;
|
|
14
|
+
const key = line.substring(0, colonIdx).trim().toLowerCase();
|
|
15
|
+
const value = line.substring(colonIdx + 1).trim();
|
|
16
|
+
if (key === "user-agent") {
|
|
17
|
+
current = { userAgent: value, allow: [], disallow: [] };
|
|
18
|
+
rules.push(current);
|
|
19
|
+
}
|
|
20
|
+
else if (key === "allow" && current) {
|
|
21
|
+
current.allow.push(value);
|
|
22
|
+
}
|
|
23
|
+
else if (key === "disallow" && current) {
|
|
24
|
+
current.disallow.push(value);
|
|
25
|
+
}
|
|
26
|
+
else if (key === "crawl-delay" && current) {
|
|
27
|
+
current.crawlDelay = parseFloat(value);
|
|
28
|
+
}
|
|
29
|
+
else if (key === "sitemap") {
|
|
30
|
+
sitemaps.push(value);
|
|
31
|
+
}
|
|
32
|
+
else if (key === "host") {
|
|
33
|
+
host = value;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return { rules, sitemaps, host };
|
|
37
|
+
};
|
|
38
|
+
const create = (args) => {
|
|
39
|
+
const config = (typeof args[0] === "object" && args[0] !== null ? args[0] : {});
|
|
40
|
+
const lines = [];
|
|
41
|
+
const rules = config.rules ?? [];
|
|
42
|
+
for (const rule of rules) {
|
|
43
|
+
lines.push(`User-agent: ${rule.userAgent}`);
|
|
44
|
+
for (const d of rule.disallow ?? [])
|
|
45
|
+
lines.push(`Disallow: ${d}`);
|
|
46
|
+
for (const a of rule.allow ?? [])
|
|
47
|
+
lines.push(`Allow: ${a}`);
|
|
48
|
+
if (rule.crawlDelay !== undefined)
|
|
49
|
+
lines.push(`Crawl-delay: ${rule.crawlDelay}`);
|
|
50
|
+
lines.push("");
|
|
51
|
+
}
|
|
52
|
+
for (const s of config.sitemaps ?? [])
|
|
53
|
+
lines.push(`Sitemap: ${s}`);
|
|
54
|
+
if (config.host)
|
|
55
|
+
lines.push(`Host: ${config.host}`);
|
|
56
|
+
return lines.join("\n").trim();
|
|
57
|
+
};
|
|
58
|
+
function pathMatches(pattern, path) {
|
|
59
|
+
if (!pattern)
|
|
60
|
+
return false;
|
|
61
|
+
let re = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
62
|
+
if (re.endsWith("$"))
|
|
63
|
+
re = re;
|
|
64
|
+
else
|
|
65
|
+
re = re + ".*";
|
|
66
|
+
return new RegExp("^" + re).test(path);
|
|
67
|
+
}
|
|
68
|
+
const isAllowed = (args) => {
|
|
69
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
70
|
+
const url = String(args[1] ?? "/");
|
|
71
|
+
const agent = String(args[2] ?? "*").toLowerCase();
|
|
72
|
+
const path = url.startsWith("http") ? new URL(url).pathname : url;
|
|
73
|
+
const applicableRules = robotsTxt.rules.filter((r) => r.userAgent === "*" || r.userAgent.toLowerCase() === agent);
|
|
74
|
+
if (applicableRules.length === 0)
|
|
75
|
+
return true;
|
|
76
|
+
let bestMatch = "";
|
|
77
|
+
let allowed = true;
|
|
78
|
+
for (const rule of applicableRules) {
|
|
79
|
+
for (const pattern of rule.allow) {
|
|
80
|
+
if (pathMatches(pattern, path) && pattern.length > bestMatch.length) {
|
|
81
|
+
bestMatch = pattern;
|
|
82
|
+
allowed = true;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
for (const pattern of rule.disallow) {
|
|
86
|
+
if (pathMatches(pattern, path) && pattern.length > bestMatch.length) {
|
|
87
|
+
bestMatch = pattern;
|
|
88
|
+
allowed = false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return allowed;
|
|
93
|
+
};
|
|
94
|
+
const isDisallowed = (args) => !isAllowed(args);
|
|
95
|
+
const getCrawlDelay = (args) => {
|
|
96
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
97
|
+
const agent = String(args[1] ?? "*").toLowerCase();
|
|
98
|
+
for (const rule of robotsTxt.rules) {
|
|
99
|
+
if (rule.userAgent.toLowerCase() === agent && rule.crawlDelay !== undefined)
|
|
100
|
+
return rule.crawlDelay;
|
|
101
|
+
}
|
|
102
|
+
for (const rule of robotsTxt.rules) {
|
|
103
|
+
if (rule.userAgent === "*" && rule.crawlDelay !== undefined)
|
|
104
|
+
return rule.crawlDelay;
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
};
|
|
108
|
+
const getSitemaps = (args) => {
|
|
109
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
110
|
+
return robotsTxt.sitemaps;
|
|
111
|
+
};
|
|
112
|
+
const getRules = (args) => {
|
|
113
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
114
|
+
const agent = args[1] ? String(args[1]).toLowerCase() : undefined;
|
|
115
|
+
if (!agent)
|
|
116
|
+
return robotsTxt.rules;
|
|
117
|
+
return robotsTxt.rules.filter((r) => r.userAgent === "*" || r.userAgent.toLowerCase() === agent);
|
|
118
|
+
};
|
|
119
|
+
const addRule = (args) => {
|
|
120
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
121
|
+
const rule = (typeof args[1] === "object" && args[1] !== null ? args[1] : {});
|
|
122
|
+
const newRule = { userAgent: rule.userAgent ?? "*", allow: rule.allow ?? [], disallow: rule.disallow ?? [], crawlDelay: rule.crawlDelay };
|
|
123
|
+
return { ...robotsTxt, rules: [...robotsTxt.rules, newRule] };
|
|
124
|
+
};
|
|
125
|
+
const removeRule = (args) => {
|
|
126
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
127
|
+
const agent = String(args[1] ?? "*").toLowerCase();
|
|
128
|
+
return { ...robotsTxt, rules: robotsTxt.rules.filter((r) => r.userAgent.toLowerCase() !== agent) };
|
|
129
|
+
};
|
|
130
|
+
const addSitemap = (args) => {
|
|
131
|
+
const robotsTxt = (typeof args[0] === "object" && args[0] !== null ? args[0] : parse([String(args[0] ?? "")]));
|
|
132
|
+
const url = String(args[1] ?? "");
|
|
133
|
+
if (robotsTxt.sitemaps.includes(url))
|
|
134
|
+
return robotsTxt;
|
|
135
|
+
return { ...robotsTxt, sitemaps: [...robotsTxt.sitemaps, url] };
|
|
136
|
+
};
|
|
137
|
+
const allowAll = (args) => {
|
|
138
|
+
const agent = String(args[0] ?? "*");
|
|
139
|
+
return create([{ rules: [{ userAgent: agent, allow: ["/"], disallow: [] }], sitemaps: [] }]);
|
|
140
|
+
};
|
|
141
|
+
const disallowAll = (args) => {
|
|
142
|
+
const agent = String(args[0] ?? "*");
|
|
143
|
+
return create([{ rules: [{ userAgent: agent, allow: [], disallow: ["/"] }], sitemaps: [] }]);
|
|
144
|
+
};
|
|
145
|
+
const fetchRobots = async (args) => {
|
|
146
|
+
const baseUrl = String(args[0] ?? "").replace(/\/+$/, "");
|
|
147
|
+
const url = baseUrl.includes("robots.txt") ? baseUrl : `${baseUrl}/robots.txt`;
|
|
148
|
+
const res = await fetch(url);
|
|
149
|
+
if (!res.ok)
|
|
150
|
+
return { found: false, status: res.status, parsed: null, raw: "" };
|
|
151
|
+
const text = await res.text();
|
|
152
|
+
return { found: true, status: res.status, parsed: parse([text]), raw: text };
|
|
153
|
+
};
|
|
154
|
+
export const RobotsFunctions = { parse, create, isAllowed, isDisallowed, getCrawlDelay, getSitemaps, getRules, addRule, removeRule, addSitemap, allowAll, disallowAll, fetch: fetchRobots };
|
|
155
|
+
export const RobotsFunctionMetadata = {
|
|
156
|
+
parse: { description: "Parse robots.txt content", parameters: [{ name: "text", dataType: "string", description: "robots.txt content", formInputType: "text", required: true }], returnType: "object", returnDescription: "{rules[], sitemaps[], host}", example: 'robots.parse $text' },
|
|
157
|
+
create: { description: "Generate robots.txt", parameters: [{ name: "config", dataType: "object", description: "{rules[], sitemaps[], host}", formInputType: "text", required: true }], returnType: "string", returnDescription: "robots.txt content", example: 'robots.create {"rules": [{"userAgent": "*", "disallow": ["/admin"]}]}' },
|
|
158
|
+
isAllowed: { description: "Check if URL is allowed", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots or raw text", formInputType: "text", required: true }, { name: "url", dataType: "string", description: "URL or path", formInputType: "text", required: true }, { name: "userAgent", dataType: "string", description: "Bot name (default *)", formInputType: "text", required: false }], returnType: "boolean", returnDescription: "true if allowed", example: 'robots.isAllowed $robots "/page" "Googlebot"' },
|
|
159
|
+
isDisallowed: { description: "Check if URL is disallowed", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots or raw text", formInputType: "text", required: true }, { name: "url", dataType: "string", description: "URL or path", formInputType: "text", required: true }, { name: "userAgent", dataType: "string", description: "Bot name", formInputType: "text", required: false }], returnType: "boolean", returnDescription: "true if disallowed", example: 'robots.isDisallowed $robots "/admin"' },
|
|
160
|
+
getCrawlDelay: { description: "Get crawl delay for agent", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }, { name: "userAgent", dataType: "string", description: "Bot name", formInputType: "text", required: false }], returnType: "number", returnDescription: "Delay in seconds or null", example: 'robots.getCrawlDelay $robots "Googlebot"' },
|
|
161
|
+
getSitemaps: { description: "Get sitemap URLs", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }], returnType: "array", returnDescription: "Sitemap URLs", example: 'robots.getSitemaps $robots' },
|
|
162
|
+
getRules: { description: "Get rules for agent", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }, { name: "userAgent", dataType: "string", description: "Bot name (all if omitted)", formInputType: "text", required: false }], returnType: "array", returnDescription: "Matching rules", example: 'robots.getRules $robots "Googlebot"' },
|
|
163
|
+
addRule: { description: "Add rule to robots config", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }, { name: "rule", dataType: "object", description: "{userAgent, allow[], disallow[], crawlDelay}", formInputType: "text", required: true }], returnType: "object", returnDescription: "Updated config", example: 'robots.addRule $robots {"userAgent": "BadBot", "disallow": ["/"]}' },
|
|
164
|
+
removeRule: { description: "Remove rule by user agent", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }, { name: "userAgent", dataType: "string", description: "Bot name", formInputType: "text", required: true }], returnType: "object", returnDescription: "Updated config", example: 'robots.removeRule $robots "BadBot"' },
|
|
165
|
+
addSitemap: { description: "Add sitemap URL", parameters: [{ name: "robots", dataType: "object", description: "Parsed robots", formInputType: "text", required: true }, { name: "url", dataType: "string", description: "Sitemap URL", formInputType: "text", required: true }], returnType: "object", returnDescription: "Updated config", example: 'robots.addSitemap $robots "https://example.com/sitemap.xml"' },
|
|
166
|
+
allowAll: { description: "Generate allow-all robots.txt", parameters: [{ name: "userAgent", dataType: "string", description: "Bot name (default *)", formInputType: "text", required: false }], returnType: "string", returnDescription: "robots.txt", example: 'robots.allowAll' },
|
|
167
|
+
disallowAll: { description: "Generate disallow-all robots.txt", parameters: [{ name: "userAgent", dataType: "string", description: "Bot name (default *)", formInputType: "text", required: false }], returnType: "string", returnDescription: "robots.txt", example: 'robots.disallowAll' },
|
|
168
|
+
fetch: { description: "Fetch and parse robots.txt from URL", parameters: [{ name: "url", dataType: "string", description: "Site URL", formInputType: "text", required: true }], returnType: "object", returnDescription: "{found, status, parsed, raw}", example: 'robots.fetch "https://example.com"' },
|
|
169
|
+
};
|
|
170
|
+
export const RobotsModuleMetadata = {
|
|
171
|
+
description: "robots.txt parsing, generation, URL permission checking, and crawl configuration",
|
|
172
|
+
methods: ["parse", "create", "isAllowed", "isDisallowed", "getCrawlDelay", "getSitemaps", "getRules", "addRule", "removeRule", "addSitemap", "allowAll", "disallowAll", "fetch"],
|
|
173
|
+
};
|
|
174
|
+
//# sourceMappingURL=robots.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAeA,MAAM,KAAK,GAAmB,CAAC,IAAI,EAAE,EAAE;IACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACnC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,IAAwB,CAAC;IAC7B,IAAI,OAAO,GAAsB,IAAI,CAAC;IACtC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC;YAAE,SAAS;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,OAAO,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YACxD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;aAAM,IAAI,GAAG,KAAK,OAAO,IAAI,OAAO,EAAE,CAAC;YACtC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;aAAM,IAAI,GAAG,KAAK,UAAU,IAAI,OAAO,EAAE,CAAC;YACzC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,GAAG,KAAK,aAAa,IAAI,OAAO,EAAE,CAAC;YAC5C,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;aAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAgD,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,MAAM,GAAmB,CAAC,IAAI,EAAE,EAAE;IACtC,MAAM,MAAM,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAwB,CAAC;IACvG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAClE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5D,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACjF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACnE,IAAI,MAAM,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACjC,CAAC,CAAC;AAEF,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY;IAChD,IAAI,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC3B,IAAI,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC3E,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,EAAE,GAAG,EAAE,CAAC;;QACzB,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACpB,OAAO,IAAI,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,SAAS,GAAmB,CAAC,IAAI,EAAE,EAAE;IACzC,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC;IAClE,MAAM,eAAe,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,IAAI,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC;IACvH,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,SAAS,GAAG,EAAE,CAAC;IACnB,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;gBAAC,SAAS,GAAG,OAAO,CAAC;gBAAC,OAAO,GAAG,IAAI,CAAC;YAAC,CAAC;QAC/G,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;gBAAC,SAAS,GAAG,OAAO,CAAC;gBAAC,OAAO,GAAG,KAAK,CAAC;YAAC,CAAC;QAChH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAmB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAEhE,MAAM,aAAa,GAAmB,CAAC,IAAI,EAAE,EAAE;IAC7C,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;IACtG,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,SAAS,KAAK,GAAG,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC;IACtF,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,WAAW,GAAmB,CAAC,IAAI,EAAE,EAAE;IAC3C,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,OAAO,SAAS,CAAC,QAAQ,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAmB,CAAC,IAAI,EAAE,EAAE;IACxC,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAClE,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC,KAAK,CAAC;IACnC,OAAO,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,IAAI,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,CAAC;AACxG,CAAC,CAAC;AAEF,MAAM,OAAO,GAAmB,CAAC,IAAI,EAAE,EAAE;IACvC,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,IAAI,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAwB,CAAC;IACrG,MAAM,OAAO,GAAe,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IACtJ,OAAO,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,CAAC,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;AAChE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAmB,CAAC,IAAI,EAAE,EAAE;IAC1C,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IACnD,OAAO,EAAE,GAAG,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,EAAE,CAAC;AAC1G,CAAC,CAAC;AAEF,MAAM,UAAU,GAAmB,CAAC,IAAI,EAAE,EAAE;IAC1C,MAAM,SAAS,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAe,CAAC;IAC7H,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACvD,OAAO,EAAE,GAAG,SAAS,EAAE,QAAQ,EAAE,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC;AAClE,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAmB,CAAC,IAAI,EAAE,EAAE;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,WAAW,GAAmB,CAAC,IAAI,EAAE,EAAE;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,WAAW,GAAmB,KAAK,EAAE,IAAI,EAAE,EAAE;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,aAAa,CAAC;IAC/E,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;IAChF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAC/E,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAmC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;AAE5N,MAAM,CAAC,MAAM,sBAAsB,GAAG;IACpC,KAAK,EAAE,EAAE,WAAW,EAAE,0BAA0B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,6BAA6B,EAAE,OAAO,EAAE,oBAAoB,EAAE;IACvR,MAAM,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,6BAA6B,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,OAAO,EAAE,uEAAuE,EAAE;IACxU,SAAS,EAAE,EAAE,WAAW,EAAE,yBAAyB,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,2BAA2B,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,OAAO,EAAE,8CAA8C,EAAE;IAClhB,YAAY,EAAE,EAAE,WAAW,EAAE,4BAA4B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,2BAA2B,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,OAAO,EAAE,sCAAsC,EAAE;IACvgB,aAAa,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,OAAO,EAAE,0CAA0C,EAAE;IAC5Z,WAAW,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,OAAO,EAAE,4BAA4B,EAAE;IAC1Q,QAAQ,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,2BAA2B,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,OAAO,EAAE,qCAAqC,EAAE;IAClZ,OAAO,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,8CAA8C,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,OAAO,EAAE,mEAAmE,EAAE;IACnc,UAAU,EAAE,EAAE,WAAW,EAAE,2BAA2B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,OAAO,EAAE,oCAAoC,EAAE;IACxY,UAAU,EAAE,EAAE,WAAW,EAAE,iBAAiB,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,OAAO,EAAE,6DAA6D,EAAE;IACpZ,QAAQ,EAAE,EAAE,WAAW,EAAE,+BAA+B,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,EAAE;IACnR,WAAW,EAAE,EAAE,WAAW,EAAE,kCAAkC,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,sBAAsB,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,YAAY,EAAE,OAAO,EAAE,oBAAoB,EAAE;IAC5R,KAAK,EAAE,EAAE,WAAW,EAAE,qCAAqC,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,8BAA8B,EAAE,OAAO,EAAE,oCAAoC,EAAE;CACzS,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,WAAW,EAAE,kFAAkF;IAC/F,OAAO,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC;CACjL,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@robinpath/robots",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"publishConfig": { "access": "public" },
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": { ".": { "import": "./dist/index.js", "types": "./dist/index.d.ts" } },
|
|
9
|
+
"files": ["dist"],
|
|
10
|
+
"scripts": { "build": "tsc", "test": "node --import tsx --test tests/*.test.ts" },
|
|
11
|
+
"peerDependencies": { "@wiredwp/robinpath": ">=0.20.0" },
|
|
12
|
+
"devDependencies": { "@wiredwp/robinpath": "^0.30.1", "tsx": "^4.19.0", "typescript": "^5.6.0" }
|
|
13
|
+
}
|