imperium-crawl 2.4.0 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +86 -9
  2. package/dist/cli.d.ts.map +1 -1
  3. package/dist/cli.js +23 -3
  4. package/dist/cli.js.map +1 -1
  5. package/dist/constants.d.ts +1 -1
  6. package/dist/constants.d.ts.map +1 -1
  7. package/dist/constants.js +31 -1
  8. package/dist/constants.js.map +1 -1
  9. package/dist/flows/engine.d.ts +7 -0
  10. package/dist/flows/engine.d.ts.map +1 -0
  11. package/dist/flows/engine.js +183 -0
  12. package/dist/flows/engine.js.map +1 -0
  13. package/dist/flows/index.d.ts +6 -0
  14. package/dist/flows/index.d.ts.map +1 -0
  15. package/dist/flows/index.js +6 -0
  16. package/dist/flows/index.js.map +1 -0
  17. package/dist/flows/server.d.ts +11 -0
  18. package/dist/flows/server.d.ts.map +1 -0
  19. package/dist/flows/server.js +81 -0
  20. package/dist/flows/server.js.map +1 -0
  21. package/dist/flows/smart-target.d.ts +9 -0
  22. package/dist/flows/smart-target.d.ts.map +1 -0
  23. package/dist/flows/smart-target.js +84 -0
  24. package/dist/flows/smart-target.js.map +1 -0
  25. package/dist/flows/storage.d.ts +26 -0
  26. package/dist/flows/storage.d.ts.map +1 -0
  27. package/dist/flows/storage.js +118 -0
  28. package/dist/flows/storage.js.map +1 -0
  29. package/dist/flows/templates.d.ts +4 -0
  30. package/dist/flows/templates.d.ts.map +1 -0
  31. package/dist/flows/templates.js +35 -0
  32. package/dist/flows/templates.js.map +1 -0
  33. package/dist/flows/types.d.ts +3356 -0
  34. package/dist/flows/types.d.ts.map +1 -0
  35. package/dist/flows/types.js +133 -0
  36. package/dist/flows/types.js.map +1 -0
  37. package/dist/knowledge/store.d.ts +19 -0
  38. package/dist/knowledge/store.d.ts.map +1 -1
  39. package/dist/knowledge/store.js +63 -4
  40. package/dist/knowledge/store.js.map +1 -1
  41. package/dist/sessions/browser-connect.d.ts +30 -0
  42. package/dist/sessions/browser-connect.d.ts.map +1 -0
  43. package/dist/sessions/browser-connect.js +68 -0
  44. package/dist/sessions/browser-connect.js.map +1 -0
  45. package/dist/sessions/browser-state.d.ts +35 -0
  46. package/dist/sessions/browser-state.d.ts.map +1 -0
  47. package/dist/sessions/browser-state.js +74 -0
  48. package/dist/sessions/browser-state.js.map +1 -0
  49. package/dist/sessions/inject-cookies.d.ts +20 -0
  50. package/dist/sessions/inject-cookies.d.ts.map +1 -0
  51. package/dist/sessions/inject-cookies.js +57 -0
  52. package/dist/sessions/inject-cookies.js.map +1 -0
  53. package/dist/sessions/manager.d.ts +11 -1
  54. package/dist/sessions/manager.d.ts.map +1 -1
  55. package/dist/sessions/manager.js +40 -6
  56. package/dist/sessions/manager.js.map +1 -1
  57. package/dist/snapshot/store.d.ts +8 -0
  58. package/dist/snapshot/store.d.ts.map +1 -1
  59. package/dist/snapshot/store.js +48 -0
  60. package/dist/snapshot/store.js.map +1 -1
  61. package/dist/stealth/antibot-detector.d.ts +1 -1
  62. package/dist/stealth/antibot-detector.d.ts.map +1 -1
  63. package/dist/stealth/antibot-detector.js +56 -0
  64. package/dist/stealth/antibot-detector.js.map +1 -1
  65. package/dist/stealth/browser-image-extract.d.ts +43 -0
  66. package/dist/stealth/browser-image-extract.d.ts.map +1 -0
  67. package/dist/stealth/browser-image-extract.js +268 -0
  68. package/dist/stealth/browser-image-extract.js.map +1 -0
  69. package/dist/stealth/browser.d.ts +5 -0
  70. package/dist/stealth/browser.d.ts.map +1 -1
  71. package/dist/stealth/browser.js +82 -1
  72. package/dist/stealth/browser.js.map +1 -1
  73. package/dist/stealth/chrome-profile.d.ts +1 -0
  74. package/dist/stealth/chrome-profile.d.ts.map +1 -1
  75. package/dist/stealth/chrome-profile.js +28 -5
  76. package/dist/stealth/chrome-profile.js.map +1 -1
  77. package/dist/stealth/detector.d.ts +10 -1
  78. package/dist/stealth/detector.d.ts.map +1 -1
  79. package/dist/stealth/detector.js +117 -25
  80. package/dist/stealth/detector.js.map +1 -1
  81. package/dist/stealth/headers.d.ts +1 -1
  82. package/dist/stealth/headers.d.ts.map +1 -1
  83. package/dist/stealth/headers.js +94 -2
  84. package/dist/stealth/headers.js.map +1 -1
  85. package/dist/stealth/index.d.ts +4 -0
  86. package/dist/stealth/index.d.ts.map +1 -1
  87. package/dist/stealth/index.js +207 -25
  88. package/dist/stealth/index.js.map +1 -1
  89. package/dist/stealth/proxy.d.ts +40 -1
  90. package/dist/stealth/proxy.d.ts.map +1 -1
  91. package/dist/stealth/proxy.js +90 -6
  92. package/dist/stealth/proxy.js.map +1 -1
  93. package/dist/tools/action-executor.d.ts +2 -0
  94. package/dist/tools/action-executor.d.ts.map +1 -1
  95. package/dist/tools/action-executor.js +38 -0
  96. package/dist/tools/action-executor.js.map +1 -1
  97. package/dist/tools/batch-download.d.ts +33 -0
  98. package/dist/tools/batch-download.d.ts.map +1 -0
  99. package/dist/tools/batch-download.js +208 -0
  100. package/dist/tools/batch-download.js.map +1 -0
  101. package/dist/tools/browser.d.ts +100 -0
  102. package/dist/tools/browser.d.ts.map +1 -0
  103. package/dist/tools/browser.js +448 -0
  104. package/dist/tools/browser.js.map +1 -0
  105. package/dist/tools/download.d.ts +35 -2
  106. package/dist/tools/download.d.ts.map +1 -1
  107. package/dist/tools/download.js +245 -44
  108. package/dist/tools/download.js.map +1 -1
  109. package/dist/tools/index.d.ts.map +1 -1
  110. package/dist/tools/index.js +23 -0
  111. package/dist/tools/index.js.map +1 -1
  112. package/dist/tools/inspect-flow.d.ts +24 -0
  113. package/dist/tools/inspect-flow.d.ts.map +1 -0
  114. package/dist/tools/inspect-flow.js +23 -0
  115. package/dist/tools/inspect-flow.js.map +1 -0
  116. package/dist/tools/interact.d.ts +28 -15
  117. package/dist/tools/interact.d.ts.map +1 -1
  118. package/dist/tools/interact.js +48 -1
  119. package/dist/tools/interact.js.map +1 -1
  120. package/dist/tools/list-flows.d.ts +21 -0
  121. package/dist/tools/list-flows.d.ts.map +1 -0
  122. package/dist/tools/list-flows.js +18 -0
  123. package/dist/tools/list-flows.js.map +1 -0
  124. package/dist/tools/manifest.d.ts.map +1 -1
  125. package/dist/tools/manifest.js +43 -0
  126. package/dist/tools/manifest.js.map +1 -1
  127. package/dist/tools/monitor.d.ts +46 -0
  128. package/dist/tools/monitor.d.ts.map +1 -0
  129. package/dist/tools/monitor.js +213 -0
  130. package/dist/tools/monitor.js.map +1 -0
  131. package/dist/tools/pdf-extract.d.ts +38 -0
  132. package/dist/tools/pdf-extract.d.ts.map +1 -0
  133. package/dist/tools/pdf-extract.js +244 -0
  134. package/dist/tools/pdf-extract.js.map +1 -0
  135. package/dist/tools/record-flow.d.ts +39 -0
  136. package/dist/tools/record-flow.d.ts.map +1 -0
  137. package/dist/tools/record-flow.js +406 -0
  138. package/dist/tools/record-flow.js.map +1 -0
  139. package/dist/tools/run-flow.d.ts +54 -0
  140. package/dist/tools/run-flow.d.ts.map +1 -0
  141. package/dist/tools/run-flow.js +47 -0
  142. package/dist/tools/run-flow.js.map +1 -0
  143. package/dist/tools/run-skill.d.ts +2 -2
  144. package/dist/tools/run-skill.d.ts.map +1 -1
  145. package/dist/tools/run-skill.js +1 -0
  146. package/dist/tools/run-skill.js.map +1 -1
  147. package/dist/tools/scrape.d.ts.map +1 -1
  148. package/dist/tools/scrape.js +17 -1
  149. package/dist/tools/scrape.js.map +1 -1
  150. package/dist/tools/serve-flow.d.ts +36 -0
  151. package/dist/tools/serve-flow.d.ts.map +1 -0
  152. package/dist/tools/serve-flow.js +42 -0
  153. package/dist/tools/serve-flow.js.map +1 -0
  154. package/dist/tools/validate-flow.d.ts +24 -0
  155. package/dist/tools/validate-flow.d.ts.map +1 -0
  156. package/dist/tools/validate-flow.js +23 -0
  157. package/dist/tools/validate-flow.js.map +1 -0
  158. package/dist/tools/watch.d.ts +68 -0
  159. package/dist/tools/watch.d.ts.map +1 -0
  160. package/dist/tools/watch.js +224 -0
  161. package/dist/tools/watch.js.map +1 -0
  162. package/dist/utils/fetcher.d.ts +13 -4
  163. package/dist/utils/fetcher.d.ts.map +1 -1
  164. package/dist/utils/fetcher.js +121 -24
  165. package/dist/utils/fetcher.js.map +1 -1
  166. package/package.json +15 -4
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/flows/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,YAAY,QAAqB,CAAC;AAE/C,eAAO,MAAM,mBAAmB,8CAA4C,CAAC;AAC7E,eAAO,MAAM,iBAAiB,2CAAyC,CAAC;AACxE,eAAO,MAAM,kBAAkB,yCAAuC,CAAC;AAEvE,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;EAK1B,CAAC;AAEH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAmBnB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAWrB,CAAC;AAEjB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;gCAQnB,CAAC;AAEjB,eAAO,MAAM,yBAAyB;;;;;;;;;;;;gCAItB,CAAC;AAEjB,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkB9B,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCA0BX,CAAC;AAEjB,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;GAMnB,CAAC;AAEf,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqBZ,CAAC;AAEZ,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAC9D,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACtE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,UAAU,CAAC,CAAC;AAExD,MAAM,WAAW,OAAO;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAC;IACxD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,OAAO,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,QAAQ,EAAE,YAAY,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH"}
@@ -0,0 +1,133 @@
1
+ import { z } from "zod";
2
+ export const FLOW_NAME_RE = /^[a-zA-Z0-9_-]+$/;
3
+ export const captchaPolicySchema = z.enum(["auto", "manual", "off", "fail"]);
4
+ export const browserModeSchema = z.enum(["auto", "headed", "headless"]);
5
+ export const evidenceModeSchema = z.enum(["off", "configured", "all"]);
6
+ export const flowInputSchema = z.object({
7
+ description: z.string().optional(),
8
+ required: z.boolean().default(false).optional(),
9
+ default: z.string().optional(),
10
+ secret: z.boolean().default(false).optional(),
11
+ });
12
+ export const smartTargetSchema = z.object({
13
+ tag: z.string().optional(),
14
+ role: z.string().optional(),
15
+ name: z.string().optional(),
16
+ text: z.string().optional(),
17
+ label: z.string().optional(),
18
+ selector: z.string().optional(),
19
+ xpath: z.string().optional(),
20
+ attributes: z.record(z.string()).optional(),
21
+ nearby_text: z.string().optional(),
22
+ href: z.string().optional(),
23
+ input_type: z.string().optional(),
24
+ form_selector: z.string().optional(),
25
+ x: z.number().optional(),
26
+ y: z.number().optional(),
27
+ viewport: z.object({
28
+ width: z.number(),
29
+ height: z.number(),
30
+ }).optional(),
31
+ }).strict();
32
+ export const flowRecordingEventSchema = z.object({
33
+ type: z.string(),
34
+ timestamp: z.string(),
35
+ page_url: z.string().optional(),
36
+ title: z.string().optional(),
37
+ target: smartTargetSchema.optional(),
38
+ value: z.string().optional(),
39
+ checked: z.boolean().optional(),
40
+ selected_text: z.string().optional(),
41
+ key: z.string().optional(),
42
+ submitter: smartTargetSchema.optional(),
43
+ }).passthrough();
44
+ export const flowNetworkEventSchema = z.object({
45
+ type: z.enum(["request", "response"]),
46
+ timestamp: z.string(),
47
+ url: z.string(),
48
+ method: z.string().optional(),
49
+ resource_type: z.string().optional(),
50
+ status: z.number().optional(),
51
+ content_type: z.string().optional(),
52
+ }).passthrough();
53
+ export const flowNavigationEventSchema = z.object({
54
+ timestamp: z.string(),
55
+ url: z.string(),
56
+ frame_url: z.string().optional(),
57
+ }).passthrough();
58
+ export const flowRecordingSchema = z.object({
59
+ started_at: z.string(),
60
+ stopped_at: z.string().optional(),
61
+ start_url: z.string(),
62
+ user_agent: z.string().optional(),
63
+ viewport: z.object({
64
+ width: z.number(),
65
+ height: z.number(),
66
+ }).optional(),
67
+ events: z.array(flowRecordingEventSchema).default([]),
68
+ network: z.array(flowNetworkEventSchema).default([]),
69
+ navigations: z.array(flowNavigationEventSchema).default([]),
70
+ }).default({
71
+ started_at: new Date(0).toISOString(),
72
+ start_url: "",
73
+ events: [],
74
+ network: [],
75
+ navigations: [],
76
+ });
77
+ export const flowStepSchema = z.object({
78
+ id: z.string().optional(),
79
+ type: z.string(),
80
+ selector: z.string().optional(),
81
+ ref: z.string().optional(),
82
+ target: smartTargetSchema.optional(),
83
+ text: z.string().optional(),
84
+ value: z.string().optional(),
85
+ script: z.string().optional(),
86
+ key: z.string().optional(),
87
+ url: z.string().optional(),
88
+ duration: z.number().optional(),
89
+ x: z.number().optional(),
90
+ y: z.number().optional(),
91
+ target_selector: z.string().optional(),
92
+ target_ref: z.string().optional(),
93
+ file_paths: z.array(z.string()).optional(),
94
+ storage: z.enum(["local", "session"]).optional(),
95
+ auth_profile: z.string().optional(),
96
+ next_selector: z.string().optional(),
97
+ next_ref: z.string().optional(),
98
+ extract_script: z.string().optional(),
99
+ max_pages: z.number().optional(),
100
+ wait_after_click: z.number().optional(),
101
+ output: z.string().optional(),
102
+ output_from: z.enum(["result", "page_text", "page_html", "url"]).optional(),
103
+ }).passthrough();
104
+ export const evidenceConfigSchema = z.object({
105
+ screenshots: z.boolean().default(false).optional(),
106
+ html: z.boolean().default(false).optional(),
107
+ markdown: z.boolean().default(false).optional(),
108
+ network_log: z.boolean().default(false).optional(),
109
+ action_log: z.boolean().default(true).optional(),
110
+ }).default({});
111
+ export const flowSchema = z.object({
112
+ schema_version: z.literal(1),
113
+ family: z.string().regex(FLOW_NAME_RE),
114
+ variant: z.string().regex(FLOW_NAME_RE),
115
+ name: z.string().optional(),
116
+ description: z.string().optional(),
117
+ url: z.string().min(1),
118
+ created_at: z.string(),
119
+ updated_at: z.string().optional(),
120
+ inputs: z.record(flowInputSchema).default({}),
121
+ outputs: z.record(z.string()).default({}).optional(),
122
+ steps: z.array(flowStepSchema).default([]),
123
+ recording: flowRecordingSchema.optional(),
124
+ captcha: captchaPolicySchema.default("auto").optional(),
125
+ browser: browserModeSchema.default("auto").optional(),
126
+ evidence: evidenceConfigSchema.optional(),
127
+ session_id: z.string().optional(),
128
+ allowed_domains: z.array(z.string()).optional(),
129
+ timeout: z.number().min(1000).default(30_000).optional(),
130
+ proxy: z.string().optional(),
131
+ chrome_profile: z.string().optional(),
132
+ }).strict();
133
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/flows/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,YAAY,GAAG,kBAAkB,CAAC;AAE/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC;AACxE,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC;AAEvE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;CAC9C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxB,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;KACnB,CAAC,CAAC,QAAQ,EAAE;CACd,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,MAAM,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACpC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC/B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,SAAS,EAAE,iBAAiB,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC,WAAW,EAAE,CAAC;AAEjB,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACpC,CAAC,CAAC,WAAW,EAAE,CAAC;AAEjB,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;IACf,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC,WAAW,EAAE,CAAC;AAEjB,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC;QACjB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;KACnB,CAAC,CAAC,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACrD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACpD,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAC5D,CAAC,CAAC,OAAO,CAAC;IACT,UAAU,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;IACrC,SAAS,EAAE,EAAE;IACb,MAAM,EAAE,EAAE;IACV,OAAO,EAAE,EAAE;IACX,WAAW,EAAE,EAAE;CAChB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,MAAM,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxB,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACtC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC1C,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACnC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC5E,CAAC,CAAC,WAAW,EAAE,CAAC;AAEjB,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAClD,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAC3C,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAC/C,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;IAClD,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAEf,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC;IACtC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC;IACvC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpD,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1C,SAAS,EAAE,mBAAmB,CAAC,QAAQ,EAAE;IACzC,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IACvD,OAAO,EAAE,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IACzC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;IACxD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC,MAAM,EAAE,CAAC"}
@@ -1,6 +1,8 @@
1
1
  import { type DomainKnowledge, type FetchOutcome, type PredictedConfig } from "./predictor.js";
2
2
  export declare class AdaptiveLearningEngine {
3
3
  private store;
4
+ /** Index: anti-bot system → Set of domains using it */
5
+ private antiBotIndex;
4
6
  private dirty;
5
7
  private writeTimer;
6
8
  private filePath;
@@ -9,11 +11,28 @@ export declare class AdaptiveLearningEngine {
9
11
  constructor(filePath?: string);
10
12
  private ensureLoaded;
11
13
  load(): Promise<void>;
14
+ /**
15
+ * Index a domain by its anti-bot system for cross-domain lookup.
16
+ */
17
+ private indexAntiBot;
18
+ /**
19
+ * Extract parent domain from a subdomain.
20
+ * e.g., "shop.example.com" → "example.com"
21
+ */
22
+ private getParentDomain;
12
23
  save(): Promise<void>;
13
24
  private scheduleSave;
14
25
  flush(): Promise<void>;
15
26
  get(domain: string): Promise<DomainKnowledge | null>;
27
+ /**
28
+ * Predict optimal fetch configuration for a URL.
29
+ * Falls back to parent domain knowledge with reduced confidence.
30
+ */
16
31
  predict(url: string): Promise<PredictedConfig | null>;
32
+ /**
33
+ * Find domains known to use a specific anti-bot system.
34
+ */
35
+ getDomainsWithAntiBot(system: string): Promise<string[]>;
17
36
  record(outcome: FetchOutcome): Promise<void>;
18
37
  prune(): void;
19
38
  /** Exposed for testing */
@@ -1 +1 @@
1
- {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/knowledge/store.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,eAAe,EAGrB,MAAM,gBAAgB,CAAC;AAUxB,qBAAa,sBAAsB;IACjC,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA8B;gBAEjC,QAAQ,CAAC,EAAE,MAAM;YAMf,YAAY;IASpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BrB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B,OAAO,CAAC,YAAY;IAed,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAYtB,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAKpD,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAUrD,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAWlD,KAAK,IAAI,IAAI;IA4Bb,0BAA0B;IAC1B,IAAI,IAAI,IAAI,MAAM,CAEjB;CACF;AAMD,wBAAgB,kBAAkB,IAAI,sBAAsB,CAK3D;AAED,oCAAoC;AACpC,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C"}
1
+ {"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/knowledge/store.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,eAAe,EAGrB,MAAM,gBAAgB,CAAC;AAWxB,qBAAa,sBAAsB;IACjC,OAAO,CAAC,KAAK,CAAsC;IACnD,uDAAuD;IACvD,OAAO,CAAC,YAAY,CAAkC;IACtD,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA8B;gBAEjC,QAAQ,CAAC,EAAE,MAAM;YAMf,YAAY;IASpB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B3B;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAQjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB3B,OAAO,CAAC,YAAY;IAed,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAYtB,GAAG,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAK1D;;;OAGG;IACG,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IA0B3D;;OAEG;IACG,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQxD,MAAM,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAelD,KAAK,IAAI,IAAI;IAkCb,0BAA0B;IAC1B,IAAI,IAAI,IAAI,MAAM,CAEjB;CACF;AAMD,wBAAgB,kBAAkB,IAAI,sBAAsB,CAK3D;AAED,oCAAoC;AACpC,wBAAgB,oBAAoB,IAAI,IAAI,CAE3C"}
@@ -7,9 +7,12 @@ import { predict, aggregateOutcome, } from "./predictor.js";
7
7
  const DEBOUNCE_MS = 30_000;
8
8
  const MAX_DOMAINS = 2000;
9
9
  const PRUNE_AGE_DAYS = 30;
10
+ const PARENT_DOMAIN_CONFIDENCE_MULTIPLIER = 0.5;
10
11
  // ── AdaptiveLearningEngine ──
11
12
  export class AdaptiveLearningEngine {
12
13
  store = new Map();
14
+ /** Index: anti-bot system → Set of domains using it */
15
+ antiBotIndex = new Map();
13
16
  dirty = false;
14
17
  writeTimer = null;
15
18
  filePath;
@@ -33,8 +36,10 @@ export class AdaptiveLearningEngine {
33
36
  const data = await fs.readFile(this.filePath, "utf-8");
34
37
  const parsed = JSON.parse(data);
35
38
  this.store.clear();
39
+ this.antiBotIndex.clear();
36
40
  for (const [domain, knowledge] of Object.entries(parsed)) {
37
41
  this.store.set(domain, knowledge);
42
+ this.indexAntiBot(domain, knowledge);
38
43
  }
39
44
  // Prune on load if over limit
40
45
  if (this.store.size > MAX_DOMAINS) {
@@ -47,10 +52,31 @@ export class AdaptiveLearningEngine {
47
52
  console.error("[knowledge] Failed to load knowledge file:", err instanceof Error ? err.message : String(err));
48
53
  }
49
54
  this.store.clear();
55
+ this.antiBotIndex.clear();
50
56
  }
51
57
  this.loaded = true;
52
58
  this.loading = null;
53
59
  }
60
+ /**
61
+ * Index a domain by its anti-bot system for cross-domain lookup.
62
+ */
63
+ indexAntiBot(domain, knowledge) {
64
+ if (knowledge.antibot_system) {
65
+ const existing = this.antiBotIndex.get(knowledge.antibot_system) || new Set();
66
+ existing.add(domain);
67
+ this.antiBotIndex.set(knowledge.antibot_system, existing);
68
+ }
69
+ }
70
+ /**
71
+ * Extract parent domain from a subdomain.
72
+ * e.g., "shop.example.com" → "example.com"
73
+ */
74
+ getParentDomain(domain) {
75
+ const parts = domain.split(".");
76
+ if (parts.length <= 2)
77
+ return null; // Already a root domain
78
+ return parts.slice(1).join(".");
79
+ }
54
80
  // ── Persistence ──
55
81
  async save() {
56
82
  const dir = path.dirname(this.filePath);
@@ -96,13 +122,39 @@ export class AdaptiveLearningEngine {
96
122
  await this.ensureLoaded();
97
123
  return this.store.get(domain) ?? null;
98
124
  }
125
+ /**
126
+ * Predict optimal fetch configuration for a URL.
127
+ * Falls back to parent domain knowledge with reduced confidence.
128
+ */
99
129
  async predict(url) {
100
130
  await this.ensureLoaded();
101
131
  const domain = getDomain(url);
102
- const knowledge = this.store.get(domain);
103
- if (!knowledge)
104
- return null;
105
- return predict(knowledge);
132
+ // Exact domain match
133
+ const exactKnowledge = this.store.get(domain);
134
+ if (exactKnowledge) {
135
+ return predict(exactKnowledge);
136
+ }
137
+ // ── Cross-domain: parent domain fallback ──
138
+ const parentDomain = this.getParentDomain(domain);
139
+ if (parentDomain) {
140
+ const parentKnowledge = this.store.get(parentDomain);
141
+ if (parentKnowledge) {
142
+ const prediction = predict(parentKnowledge);
143
+ // Reduce confidence for parent-domain predictions
144
+ prediction.confidence = Math.round(prediction.confidence * PARENT_DOMAIN_CONFIDENCE_MULTIPLIER * 100) / 100;
145
+ prediction.reason = `parent:${parentDomain}, ${prediction.reason}`;
146
+ return prediction;
147
+ }
148
+ }
149
+ return null;
150
+ }
151
+ /**
152
+ * Find domains known to use a specific anti-bot system.
153
+ */
154
+ async getDomainsWithAntiBot(system) {
155
+ await this.ensureLoaded();
156
+ const domains = this.antiBotIndex.get(system);
157
+ return domains ? Array.from(domains) : [];
106
158
  }
107
159
  // ── Record ──
108
160
  async record(outcome) {
@@ -110,6 +162,8 @@ export class AdaptiveLearningEngine {
110
162
  const existing = this.store.get(outcome.domain) ?? null;
111
163
  const updated = aggregateOutcome(existing, outcome);
112
164
  this.store.set(outcome.domain, updated);
165
+ // Update anti-bot index
166
+ this.indexAntiBot(outcome.domain, updated);
113
167
  this.dirty = true;
114
168
  this.scheduleSave();
115
169
  }
@@ -132,6 +186,11 @@ export class AdaptiveLearningEngine {
132
186
  this.store.delete(sorted[i][0]);
133
187
  }
134
188
  }
189
+ // Rebuild anti-bot index after pruning
190
+ this.antiBotIndex.clear();
191
+ for (const [domain, knowledge] of this.store) {
192
+ this.indexAntiBot(domain, knowledge);
193
+ }
135
194
  if (this.store.size > 0) {
136
195
  this.dirty = true;
137
196
  }
@@ -1 +1 @@
1
- {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/knowledge/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAIL,OAAO,EACP,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAExB,kBAAkB;AAElB,MAAM,WAAW,GAAG,MAAM,CAAC;AAC3B,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B,+BAA+B;AAE/B,MAAM,OAAO,sBAAsB;IACzB,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IAC3C,KAAK,GAAG,KAAK,CAAC;IACd,UAAU,GAAyC,IAAI,CAAC;IACxD,QAAQ,CAAS;IACjB,MAAM,GAAG,KAAK,CAAC;IACf,OAAO,GAAyB,IAAI,CAAC;IAE7C,YAAY,QAAiB;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,oBAAoB,EAAE,CAAC;IACrD,CAAC;IAED,kBAAkB;IAEV,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,+BAA+B;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7B,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoC,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACpC,CAAC;YACD,8BAA8B;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,QAAQ,GACZ,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAK,GAAwC,CAAC,IAAI,KAAK,QAAQ,CAAC;YACjH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChH,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,oBAAoB;IAEpB,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,GAAG,GAAoC,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,GAAG,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;QAEvC,mCAAmC;QACnC,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,CAAC;gBAAC,MAAM,CAAC;oBACP,+CAA+C;gBACjD,CAAC;YACH,CAAC;QACH,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,cAAc;IAEd,KAAK,CAAC,GAAG,CAAC,MAAc;QACtB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAC5B,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5B,CAAC;IAED,eAAe;IAEf,KAAK,CAAC,MAAM,CAAC,OAAqB;QAChC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED,oBAAoB;IAEpB,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAErD,2CAA2C;QAC3C,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;YAC7D,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CACxF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;CACF;AAED,kBAAkB;AAElB,IAAI,MAAM,GAAkC,IAAI,CAAC;AAEjD,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,sBAAsB,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,oBAAoB;IAClC,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/knowledge/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAIL,OAAO,EACP,gBAAgB,GACjB,MAAM,gBAAgB,CAAC;AAExB,kBAAkB;AAElB,MAAM,WAAW,GAAG,MAAM,CAAC;AAC3B,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,cAAc,GAAG,EAAE,CAAC;AAC1B,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD,+BAA+B;AAE/B,MAAM,OAAO,sBAAsB;IACzB,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IACnD,uDAAuD;IAC/C,YAAY,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,GAAG,KAAK,CAAC;IACd,UAAU,GAAyC,IAAI,CAAC;IACxD,QAAQ,CAAS;IACjB,MAAM,GAAG,KAAK,CAAC;IACf,OAAO,GAAyB,IAAI,CAAC;IAE7C,YAAY,QAAiB;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,oBAAoB,EAAE,CAAC;IACrD,CAAC;IAED,kBAAkB;IAEV,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,+BAA+B;QAC/B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7B,CAAC;QACD,MAAM,IAAI,CAAC,OAAO,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAoC,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;YAC1B,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAClC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACvC,CAAC;YACD,8BAA8B;YAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,QAAQ,GACZ,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAK,GAAwC,CAAC,IAAI,KAAK,QAAQ,CAAC;YACjH,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,4CAA4C,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAChH,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,MAAc,EAAE,SAA0B;QAC7D,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC;YAC9E,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,MAAc;QACpC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,wBAAwB;QAC5D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,oBAAoB;IAEpB,KAAK,CAAC,IAAI;QACR,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzC,MAAM,GAAG,GAAoC,EAAE,CAAC;QAChD,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,GAAG,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC;QAEvC,mCAAmC;QACnC,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpB,CAAC;gBAAC,MAAM,CAAC;oBACP,+CAA+C;gBACjD,CAAC;YACH,CAAC;QACH,CAAC,EAAE,WAAW,CAAC,CAAC;QAChB,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,cAAc;IAEd,KAAK,CAAC,GAAG,CAAC,MAAc;QACtB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,CAAC,GAAW;QACvB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;QAE9B,qBAAqB;QACrB,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,OAAO,CAAC,cAAc,CAAC,CAAC;QACjC,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACrD,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,UAAU,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;gBAC5C,kDAAkD;gBAClD,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,GAAG,mCAAmC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;gBAC5G,UAAU,CAAC,MAAM,GAAG,UAAU,YAAY,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC;gBACnE,OAAO,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,qBAAqB,CAAC,MAAc;QACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5C,CAAC;IAED,eAAe;IAEf,KAAK,CAAC,MAAM,CAAC,OAAqB;QAChC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACpD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAExC,wBAAwB;QACxB,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED,oBAAoB;IAEpB,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAErD,2CAA2C;QAC3C,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC;YAC7D,IAAI,GAAG,GAAG,OAAO,EAAE,CAAC;gBAClB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CACxF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC7C,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;CACF;AAED,kBAAkB;AAElB,IAAI,MAAM,GAAkC,IAAI,CAAC;AAEjD,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,sBAAsB,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,oCAAoC;AACpC,MAAM,UAAU,oBAAoB;IAClC,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * CDP Reconnect helper — connects to a persistent Chrome session via CDP.
3
+ *
4
+ * The key insight: connectOverCDP returns a Browser object, but calling
5
+ * browser.close() on a CDP-connected browser only DISCONNECTS — it does
6
+ * NOT close the actual Chrome process. Chrome lives on independently.
7
+ *
8
+ * Flow:
9
+ * 1. Load browser state from disk (endpoint + PID)
10
+ * 2. Verify PID is alive
11
+ * 3. Connect via CDP
12
+ * 4. Get existing page (or create new one)
13
+ * 5. Return { browser, page, disconnect }
14
+ * 6. disconnect() syncs cookies to SessionManager, then disconnects
15
+ */
16
+ type Browser = import("rebrowser-playwright").Browser;
17
+ type Page = import("rebrowser-playwright").Page;
18
+ export interface BrowserConnection {
19
+ browser: Browser;
20
+ page: Page;
21
+ /** Disconnect from Chrome WITHOUT closing it. Syncs cookies first. */
22
+ disconnect: () => Promise<void>;
23
+ }
24
+ /**
25
+ * Connect to an existing persistent browser session via CDP.
26
+ * Throws if no browser is running for this session.
27
+ */
28
+ export declare function connectToSession(session: string): Promise<BrowserConnection>;
29
+ export {};
30
+ //# sourceMappingURL=browser-connect.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-connect.d.ts","sourceRoot":"","sources":["../../src/sessions/browser-connect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AASH,KAAK,OAAO,GAAG,OAAO,sBAAsB,EAAE,OAAO,CAAC;AACtD,KAAK,IAAI,GAAG,OAAO,sBAAsB,EAAE,IAAI,CAAC;AAEhD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,sEAAsE;IACtE,UAAU,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACjC;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAoDlF"}
@@ -0,0 +1,68 @@
1
+ /**
2
+ * CDP Reconnect helper — connects to a persistent Chrome session via CDP.
3
+ *
4
+ * The key insight: connectOverCDP returns a Browser object, but calling
5
+ * browser.close() on a CDP-connected browser only DISCONNECTS — it does
6
+ * NOT close the actual Chrome process. Chrome lives on independently.
7
+ *
8
+ * Flow:
9
+ * 1. Load browser state from disk (endpoint + PID)
10
+ * 2. Verify PID is alive
11
+ * 3. Connect via CDP
12
+ * 4. Get existing page (or create new one)
13
+ * 5. Return { browser, page, disconnect }
14
+ * 6. disconnect() syncs cookies to SessionManager, then disconnects
15
+ */
16
+ import { loadBrowserState, isBrowserAlive, clearBrowserState, } from "./browser-state.js";
17
+ import { getSessionManager } from "./manager.js";
18
+ /**
19
+ * Connect to an existing persistent browser session via CDP.
20
+ * Throws if no browser is running for this session.
21
+ */
22
+ export async function connectToSession(session) {
23
+ const state = await loadBrowserState(session);
24
+ if (!state) {
25
+ throw new Error(`No browser running for session "${session}". Run: browser start --session ${session}`);
26
+ }
27
+ const alive = await isBrowserAlive(session);
28
+ if (!alive) {
29
+ await clearBrowserState(session);
30
+ throw new Error(`Browser for session "${session}" is not running (PID ${state.pid} dead). Run: browser start --session ${session}`);
31
+ }
32
+ const { chromium } = await import("rebrowser-playwright");
33
+ const browser = await chromium.connectOverCDP(state.endpoint);
34
+ const contexts = browser.contexts();
35
+ const context = contexts[0];
36
+ if (!context) {
37
+ throw new Error(`No browser context found for session "${session}"`);
38
+ }
39
+ const pages = context.pages();
40
+ const page = pages[0] ?? await context.newPage();
41
+ return {
42
+ browser,
43
+ page,
44
+ disconnect: async () => {
45
+ try {
46
+ // Sync cookies to SessionManager before disconnecting
47
+ const cookies = await page.context().cookies();
48
+ const storedCookies = cookies.map((c) => ({
49
+ name: c.name,
50
+ value: c.value,
51
+ domain: c.domain,
52
+ path: c.path,
53
+ expires: c.expires,
54
+ httpOnly: c.httpOnly,
55
+ secure: c.secure,
56
+ sameSite: c.sameSite,
57
+ }));
58
+ await getSessionManager().save(session, storedCookies, page.url());
59
+ }
60
+ catch {
61
+ // Best effort — don't fail disconnect on cookie sync error
62
+ }
63
+ // close() on CDP-connected browser = disconnect only (Chrome stays alive)
64
+ await browser.close();
65
+ },
66
+ };
67
+ }
68
+ //# sourceMappingURL=browser-connect.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-connect.js","sourceRoot":"","sources":["../../src/sessions/browser-connect.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAYjD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAe;IACpD,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,mCAAmC,OAAO,EAAE,CAAC,CAAC;IAC1G,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACjC,MAAM,IAAI,KAAK,CACb,wBAAwB,OAAO,yBAAyB,KAAK,CAAC,GAAG,wCAAwC,OAAO,EAAE,CACnH,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,GAAG,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAEjD,OAAO;QACL,OAAO;QACP,IAAI;QACJ,UAAU,EAAE,KAAK,IAAI,EAAE;YACrB,IAAI,CAAC;gBACH,sDAAsD;gBACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACxC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAiD;iBAC9D,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACrE,CAAC;YAAC,MAAM,CAAC;gBACP,2DAA2D;YAC7D,CAAC;YAED,0EAA0E;YAC1E,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,35 @@
1
+ /**
2
+ * Browser State persistence — stores CDP endpoint + PID between CLI invocations.
3
+ *
4
+ * Each named session gets its own directory under ~/.imperium-crawl/sessions/{name}/
5
+ * with browser.json holding the CDP connection info and Chrome PID.
6
+ *
7
+ * This enables persistent browser sessions: Chrome lives independently of Node.js,
8
+ * and each CLI call reconnects via CDP (connectOverCDP).
9
+ */
10
+ export interface BrowserState {
11
+ /** CDP WebSocket endpoint, e.g. "http://127.0.0.1:9222" */
12
+ endpoint: string;
13
+ /** Chrome process ID */
14
+ pid: number;
15
+ /** ISO timestamp when browser was started */
16
+ startedAt: string;
17
+ /** Whether browser was launched with headed mode */
18
+ headed: boolean;
19
+ /** CDP port used */
20
+ port: number;
21
+ }
22
+ export declare function saveBrowserState(session: string, state: BrowserState): Promise<void>;
23
+ export declare function loadBrowserState(session: string): Promise<BrowserState | null>;
24
+ export declare function clearBrowserState(session: string): Promise<void>;
25
+ /**
26
+ * Check if the Chrome process for a session is still alive.
27
+ * Sends signal 0 (no-op) to test if PID exists.
28
+ */
29
+ export declare function isBrowserAlive(session: string): Promise<boolean>;
30
+ /**
31
+ * Get the directory path for a session's persistent data.
32
+ * Used by snapshot refs disk persistence.
33
+ */
34
+ export declare function getSessionDir(session: string): string;
35
+ //# sourceMappingURL=browser-state.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-state.d.ts","sourceRoot":"","sources":["../../src/sessions/browser-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,MAAM,WAAW,YAAY;IAC3B,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,MAAM,EAAE,OAAO,CAAC;IAChB,oBAAoB;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAWD,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAY1F;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAOpF;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAMtE;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAStE;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAErD"}
@@ -0,0 +1,74 @@
1
+ /**
2
+ * Browser State persistence — stores CDP endpoint + PID between CLI invocations.
3
+ *
4
+ * Each named session gets its own directory under ~/.imperium-crawl/sessions/{name}/
5
+ * with browser.json holding the CDP connection info and Chrome PID.
6
+ *
7
+ * This enables persistent browser sessions: Chrome lives independently of Node.js,
8
+ * and each CLI call reconnects via CDP (connectOverCDP).
9
+ */
10
+ import fs from "node:fs/promises";
11
+ import path from "node:path";
12
+ import { getSessionsDir } from "../config.js";
13
+ function browserDir(session) {
14
+ const safe = session.replace(/[^a-zA-Z0-9_\-]/g, "_");
15
+ return path.join(getSessionsDir(), safe);
16
+ }
17
+ function browserStatePath(session) {
18
+ return path.join(browserDir(session), "browser.json");
19
+ }
20
+ export async function saveBrowserState(session, state) {
21
+ const dir = browserDir(session);
22
+ await fs.mkdir(dir, { recursive: true });
23
+ const filePath = browserStatePath(session);
24
+ const tmpPath = `${filePath}.${Date.now()}.tmp`;
25
+ await fs.writeFile(tmpPath, JSON.stringify(state, null, 2), "utf-8");
26
+ try {
27
+ await fs.rename(tmpPath, filePath);
28
+ }
29
+ catch {
30
+ await fs.unlink(tmpPath).catch(() => { });
31
+ throw new Error(`Failed to save browser state for session "${session}"`);
32
+ }
33
+ }
34
+ export async function loadBrowserState(session) {
35
+ try {
36
+ const data = await fs.readFile(browserStatePath(session), "utf-8");
37
+ return JSON.parse(data);
38
+ }
39
+ catch {
40
+ return null;
41
+ }
42
+ }
43
+ export async function clearBrowserState(session) {
44
+ try {
45
+ await fs.unlink(browserStatePath(session));
46
+ }
47
+ catch {
48
+ // Already gone — fine
49
+ }
50
+ }
51
+ /**
52
+ * Check if the Chrome process for a session is still alive.
53
+ * Sends signal 0 (no-op) to test if PID exists.
54
+ */
55
+ export async function isBrowserAlive(session) {
56
+ const state = await loadBrowserState(session);
57
+ if (!state)
58
+ return false;
59
+ try {
60
+ process.kill(state.pid, 0);
61
+ return true;
62
+ }
63
+ catch {
64
+ return false;
65
+ }
66
+ }
67
+ /**
68
+ * Get the directory path for a session's persistent data.
69
+ * Used by snapshot refs disk persistence.
70
+ */
71
+ export function getSessionDir(session) {
72
+ return browserDir(session);
73
+ }
74
+ //# sourceMappingURL=browser-state.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browser-state.js","sourceRoot":"","sources":["../../src/sessions/browser-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAe9C,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe;IACvC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAe,EAAE,KAAmB;IACzE,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,6CAA6C,OAAO,GAAG,CAAC,CAAC;IAC3E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAe;IACpD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;QACnE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAAe;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,sBAAsB;IACxB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC"}
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Cookie injection helper — bridges persistent browser sessions to HTTP tools.
3
+ *
4
+ * After a browser login, cookies are stored in SessionManager.
5
+ * This module extracts those cookies and formats them as HTTP Cookie headers,
6
+ * so tools like scrape, extract, readability, download can reuse the same auth.
7
+ *
8
+ * Domain/path/expiry filtering ensures only relevant cookies are sent.
9
+ */
10
+ /**
11
+ * Get Cookie header string for a session + URL.
12
+ * Returns empty string if no matching cookies found.
13
+ */
14
+ export declare function getSessionCookieHeader(sessionId: string, url: string): Promise<string>;
15
+ /**
16
+ * Inject session cookies into HTTP request headers.
17
+ * Merges with any existing headers, adding Cookie header if session has matching cookies.
18
+ */
19
+ export declare function injectSessionHeaders(sessionId: string, url: string, headers?: Record<string, string>): Promise<Record<string, string>>;
20
+ //# sourceMappingURL=inject-cookies.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"inject-cookies.d.ts","sourceRoot":"","sources":["../../src/sessions/inject-cookies.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA6BH;;;GAGG;AACH,wBAAsB,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAI5F;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GACnC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAMjC"}