@astrofoundry/grimoire 1.9.1 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -41,5 +41,5 @@ grimoire refresh --all # refresh all sources
41
41
  grimoire export <source>
42
42
  grimoire apikey create <name>
43
43
  grimoire apikey list
44
- grimoire apikey revoke <name>
44
+ grimoire apikey delete <name>
45
45
  ```
package/dist/apikey.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export declare function createApiKey(name: string): Promise<string>;
2
2
  export declare function listApiKeys(): Promise<void>;
3
- export declare function revokeApiKey(name: string): Promise<void>;
3
+ export declare function deleteApiKey(name: string): Promise<void>;
4
4
  export declare function cmdApiKey(): Promise<void>;
5
5
  //# sourceMappingURL=apikey.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"apikey.d.ts","sourceRoot":"","sources":["../src/apikey.ts"],"names":[],"mappings":"AA4BA,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAYhE;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAiBjD;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAY9D;AAED,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAyB/C"}
1
+ {"version":3,"file":"apikey.d.ts","sourceRoot":"","sources":["../src/apikey.ts"],"names":[],"mappings":"AA4BA,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAWhE;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAgBjD;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAY9D;AAED,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAyB/C"}
package/dist/apikey.js CHANGED
@@ -25,7 +25,6 @@ export async function createApiKey(name) {
25
25
  name,
26
26
  created_at: new Date().toISOString(),
27
27
  last_used_at: null,
28
- active: true,
29
28
  });
30
29
  return raw;
31
30
  }
@@ -38,22 +37,21 @@ export async function listApiKeys() {
38
37
  console.log("\nAPI Keys:\n");
39
38
  for (const doc of snapshot.docs) {
40
39
  const data = doc.data();
41
- const status = data.active ? "active" : "revoked";
42
40
  const lastUsed = data.last_used_at ?? "never";
43
- console.log(` ${bold(data.name)} (${status})`);
41
+ console.log(` ${bold(data.name)}`);
44
42
  console.log(` Created: ${data.created_at}`);
45
43
  console.log(` Last used: ${lastUsed}`);
46
44
  }
47
45
  }
48
- export async function revokeApiKey(name) {
49
- const snapshot = await apiKeysCol().where("name", "==", name).where("active", "==", true).get();
46
+ export async function deleteApiKey(name) {
47
+ const snapshot = await apiKeysCol().where("name", "==", name).get();
50
48
  if (snapshot.empty) {
51
- throw new Error(`No active API key found with name "${name}".`);
49
+ throw new Error(`No API key found with name "${name}".`);
52
50
  }
53
51
  for (const doc of snapshot.docs) {
54
- await doc.ref.update({ active: false });
52
+ await doc.ref.delete();
55
53
  }
56
- console.log(`API key "${name}" revoked.`);
54
+ console.log(`API key "${name}" deleted.`);
57
55
  }
58
56
  export async function cmdApiKey() {
59
57
  const subcommand = process.argv[3];
@@ -71,15 +69,15 @@ export async function cmdApiKey() {
71
69
  else if (subcommand === "list") {
72
70
  await listApiKeys();
73
71
  }
74
- else if (subcommand === "revoke") {
72
+ else if (subcommand === "delete") {
75
73
  if (!arg) {
76
- console.error("Usage: grimoire apikey revoke <name>");
74
+ console.error("Usage: grimoire apikey delete <name>");
77
75
  process.exit(1);
78
76
  }
79
- await revokeApiKey(arg);
77
+ await deleteApiKey(arg);
80
78
  }
81
79
  else {
82
- console.error("Usage: grimoire apikey <create|list|revoke> [name]");
80
+ console.error("Usage: grimoire apikey <create|list|delete> [name]");
83
81
  process.exit(1);
84
82
  }
85
83
  }
@@ -1 +1 @@
1
- {"version":3,"file":"apikey.js","sourceRoot":"","sources":["../src/apikey.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,YAAY,GAEb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,IAAI,EAAyB,CAAC;AAE9B,SAAS,KAAK;IACZ,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,IAAI,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,aAAa,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,EAAE,GAAG,YAAY,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAG,QAAQ,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;IAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE1B,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;QAC/B,IAAI;QACJ,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,YAAY,EAAE,IAAI;QAClB,MAAM,EAAE,IAAI;KACb,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,EAAE,CAAC;IAE1C,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAChD,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IAEhG,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5B,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,0BAA0B,GAAG,MAAM,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC7D,CAAC;SAAM,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QACjC,MAAM,WAAW,EAAE,CAAC;IACtB,CAAC;SAAM,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"apikey.js","sourceRoot":"","sources":["../src/apikey.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,YAAY,GAEb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAEnC,IAAI,EAAyB,CAAC;AAE9B,SAAS,KAAK;IACZ,IAAI,CAAC,EAAE,EAAE,CAAC;QACR,IAAI,OAAO,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,aAAa,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;QACD,EAAE,GAAG,YAAY,EAAE,CAAC;IACtB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,GAAW;IAC1B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,KAAK,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAG,QAAQ,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;IAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IAE1B,MAAM,UAAU,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC;QAC/B,IAAI;QACJ,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,YAAY,EAAE,IAAI;KACnB,CAAC,CAAC;IAEH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,GAAG,EAAE,CAAC;IAE1C,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,IAAI,OAAO,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC7C,MAAM,QAAQ,GAAG,MAAM,UAAU,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;IAEpE,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,IAAI,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,YAAY,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS;IAC7B,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5B,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,0BAA0B,GAAG,MAAM,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC7D,CAAC;SAAM,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QACjC,MAAM,WAAW,EAAE,CAAC;IACtB,CAAC;SAAM,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;YACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
package/dist/cli.js CHANGED
@@ -446,7 +446,7 @@ Commands:
446
446
  export <source> Export source as JSON
447
447
  apikey create <name> Generate an API key
448
448
  apikey list List API keys
449
- apikey revoke <name> Revoke an API key
449
+ apikey delete <name> Delete an API key
450
450
  `);
451
451
  }
452
452
  }
@@ -1 +1 @@
1
- {"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAEA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,MAAM,EAAE,CAgBV;AASD,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBnB;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAGjB;AAsBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,MAAM,EAAE,CAAC,CAiCnB;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,CAEtD"}
1
+ {"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAEA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,MAAM,EAAE,CAkBV;AAoBD,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAanB;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAGjB;AAsBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,MAAM,EAAE,CAAC,CAiCnB;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,CAEtD"}
package/dist/scraper.js CHANGED
@@ -10,7 +10,7 @@ export function slugifyUrl(url) {
10
10
  .replace(/[^a-zA-Z0-9-]/g, "");
11
11
  }
12
12
  export function filterUrls(urls, includePatterns, excludePatterns) {
13
- let filtered = urls;
13
+ let filtered = urls.filter((url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#"));
14
14
  if (includePatterns && includePatterns.length > 0) {
15
15
  filtered = filtered.filter((url) => includePatterns.some((pattern) => url.includes(pattern)));
16
16
  }
@@ -19,10 +19,18 @@ export function filterUrls(urls, includePatterns, excludePatterns) {
19
19
  }
20
20
  return [...new Set(filtered)].sort();
21
21
  }
22
- async function discoverFromSitemap(sitemapUrl, source) {
22
+ async function fetchSitemapUrls(sitemapUrl) {
23
23
  const response = await fetch(sitemapUrl);
24
24
  const xml = await response.text();
25
- const urls = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
25
+ const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
26
+ if (xml.includes("<sitemapindex")) {
27
+ const nested = await Promise.all(locs.map((loc) => fetchSitemapUrls(loc)));
28
+ return nested.flat();
29
+ }
30
+ return locs;
31
+ }
32
+ async function discoverFromSitemap(sitemapUrl, source) {
33
+ const urls = await fetchSitemapUrls(sitemapUrl);
26
34
  return filterUrls(urls, source.include_patterns, source.exclude_patterns);
27
35
  }
28
36
  export async function discoverUrls(page, source) {
@@ -31,14 +39,13 @@ export async function discoverUrls(page, source) {
31
39
  }
32
40
  await page.goto(source.start_url, { waitUntil: "domcontentloaded" });
33
41
  const rawUrls = await page.$$eval(`${source.nav_selector} a[href]`, (links) => links.map((a) => a.href));
34
- const cleanUrls = rawUrls.filter((url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#"));
35
- return filterUrls(cleanUrls, source.include_patterns, source.exclude_patterns);
42
+ return filterUrls(rawUrls, source.include_patterns, source.exclude_patterns);
36
43
  }
37
44
  export async function fetchPage(page, url) {
38
45
  await page.goto(url, { waitUntil: "domcontentloaded" });
39
46
  return page.content();
40
47
  }
41
- const DEFAULT_CONCURRENCY = 10;
48
+ const DEFAULT_CONCURRENCY = 50;
42
49
  async function runPool(items, concurrency, fn) {
43
50
  let nextIndex = 0;
44
51
  async function worker() {
@@ -1 +1 @@
1
- {"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAG/D,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,QAAQ;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAc,EACd,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,QAAQ,GAAG,IAAI,CAAC;IAEpB,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,UAAkB,EAAE,MAAoB;IACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAU,EACV,MAAoB;IAEpB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,OAAO,mBAAmB,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAErE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,GAAG,MAAM,CAAC,YAAY,UAAU,EAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,CAAC,CAC3D,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC/E,CAAC;IAEF,OAAO,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,GAAW;IAEX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,KAAK,UAAU,OAAO,CACpB,KAAU,EACV,WAAmB,EACnB,EAA6C;IAE7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAoB,EACpB,UAAkB,EAClB,OAAe,EACf,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAE3C,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC5C,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1E,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AAChD,CAAC"}
1
+ {"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAG/D,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,QAAQ;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAc,EACd,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC/E,CAAC;IAEF,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,UAAkB;IAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,UAAkB,EAAE,MAAoB;IACzE,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAU,EACV,MAAoB;IAEpB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,OAAO,mBAAmB,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAErE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,GAAG,MAAM,CAAC,YAAY,UAAU,EAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,CAAC,CAC3D,CAAC;IAEF,OAAO,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,GAAW;IAEX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,KAAK,UAAU,OAAO,CACpB,KAAU,EACV,WAAmB,EACnB,EAA6C;IAE7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAoB,EACpB,UAAkB,EAClB,OAAe,EACf,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAE3C,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC5C,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1E,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AAChD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/grimoire",
3
- "version": "1.9.1",
3
+ "version": "1.10.0",
4
4
  "description": "Documentation RAG System",
5
5
  "keywords": [],
6
6
  "author": "",