@apitap/core 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (236) hide show
  1. package/LICENSE +60 -0
  2. package/README.md +362 -0
  3. package/SKILL.md +270 -0
  4. package/dist/auth/crypto.d.ts +31 -0
  5. package/dist/auth/crypto.js +66 -0
  6. package/dist/auth/crypto.js.map +1 -0
  7. package/dist/auth/handoff.d.ts +29 -0
  8. package/dist/auth/handoff.js +180 -0
  9. package/dist/auth/handoff.js.map +1 -0
  10. package/dist/auth/manager.d.ts +46 -0
  11. package/dist/auth/manager.js +127 -0
  12. package/dist/auth/manager.js.map +1 -0
  13. package/dist/auth/oauth-refresh.d.ts +16 -0
  14. package/dist/auth/oauth-refresh.js +91 -0
  15. package/dist/auth/oauth-refresh.js.map +1 -0
  16. package/dist/auth/refresh.d.ts +43 -0
  17. package/dist/auth/refresh.js +217 -0
  18. package/dist/auth/refresh.js.map +1 -0
  19. package/dist/capture/anti-bot.d.ts +15 -0
  20. package/dist/capture/anti-bot.js +43 -0
  21. package/dist/capture/anti-bot.js.map +1 -0
  22. package/dist/capture/blocklist.d.ts +6 -0
  23. package/dist/capture/blocklist.js +70 -0
  24. package/dist/capture/blocklist.js.map +1 -0
  25. package/dist/capture/body-diff.d.ts +8 -0
  26. package/dist/capture/body-diff.js +102 -0
  27. package/dist/capture/body-diff.js.map +1 -0
  28. package/dist/capture/body-variables.d.ts +13 -0
  29. package/dist/capture/body-variables.js +142 -0
  30. package/dist/capture/body-variables.js.map +1 -0
  31. package/dist/capture/domain.d.ts +8 -0
  32. package/dist/capture/domain.js +34 -0
  33. package/dist/capture/domain.js.map +1 -0
  34. package/dist/capture/entropy.d.ts +33 -0
  35. package/dist/capture/entropy.js +100 -0
  36. package/dist/capture/entropy.js.map +1 -0
  37. package/dist/capture/filter.d.ts +11 -0
  38. package/dist/capture/filter.js +49 -0
  39. package/dist/capture/filter.js.map +1 -0
  40. package/dist/capture/graphql.d.ts +21 -0
  41. package/dist/capture/graphql.js +99 -0
  42. package/dist/capture/graphql.js.map +1 -0
  43. package/dist/capture/idle.d.ts +23 -0
  44. package/dist/capture/idle.js +44 -0
  45. package/dist/capture/idle.js.map +1 -0
  46. package/dist/capture/monitor.d.ts +26 -0
  47. package/dist/capture/monitor.js +183 -0
  48. package/dist/capture/monitor.js.map +1 -0
  49. package/dist/capture/oauth-detector.d.ts +18 -0
  50. package/dist/capture/oauth-detector.js +96 -0
  51. package/dist/capture/oauth-detector.js.map +1 -0
  52. package/dist/capture/pagination.d.ts +9 -0
  53. package/dist/capture/pagination.js +40 -0
  54. package/dist/capture/pagination.js.map +1 -0
  55. package/dist/capture/parameterize.d.ts +17 -0
  56. package/dist/capture/parameterize.js +63 -0
  57. package/dist/capture/parameterize.js.map +1 -0
  58. package/dist/capture/scrubber.d.ts +5 -0
  59. package/dist/capture/scrubber.js +38 -0
  60. package/dist/capture/scrubber.js.map +1 -0
  61. package/dist/capture/session.d.ts +46 -0
  62. package/dist/capture/session.js +445 -0
  63. package/dist/capture/session.js.map +1 -0
  64. package/dist/capture/token-detector.d.ts +16 -0
  65. package/dist/capture/token-detector.js +62 -0
  66. package/dist/capture/token-detector.js.map +1 -0
  67. package/dist/capture/verifier.d.ts +17 -0
  68. package/dist/capture/verifier.js +147 -0
  69. package/dist/capture/verifier.js.map +1 -0
  70. package/dist/cli.d.ts +2 -0
  71. package/dist/cli.js +930 -0
  72. package/dist/cli.js.map +1 -0
  73. package/dist/discovery/auth.d.ts +17 -0
  74. package/dist/discovery/auth.js +81 -0
  75. package/dist/discovery/auth.js.map +1 -0
  76. package/dist/discovery/fetch.d.ts +17 -0
  77. package/dist/discovery/fetch.js +59 -0
  78. package/dist/discovery/fetch.js.map +1 -0
  79. package/dist/discovery/frameworks.d.ts +11 -0
  80. package/dist/discovery/frameworks.js +249 -0
  81. package/dist/discovery/frameworks.js.map +1 -0
  82. package/dist/discovery/index.d.ts +21 -0
  83. package/dist/discovery/index.js +219 -0
  84. package/dist/discovery/index.js.map +1 -0
  85. package/dist/discovery/openapi.d.ts +13 -0
  86. package/dist/discovery/openapi.js +175 -0
  87. package/dist/discovery/openapi.js.map +1 -0
  88. package/dist/discovery/probes.d.ts +9 -0
  89. package/dist/discovery/probes.js +70 -0
  90. package/dist/discovery/probes.js.map +1 -0
  91. package/dist/index.d.ts +25 -0
  92. package/dist/index.js +25 -0
  93. package/dist/index.js.map +1 -0
  94. package/dist/inspect/report.d.ts +52 -0
  95. package/dist/inspect/report.js +191 -0
  96. package/dist/inspect/report.js.map +1 -0
  97. package/dist/mcp.d.ts +8 -0
  98. package/dist/mcp.js +526 -0
  99. package/dist/mcp.js.map +1 -0
  100. package/dist/orchestration/browse.d.ts +38 -0
  101. package/dist/orchestration/browse.js +198 -0
  102. package/dist/orchestration/browse.js.map +1 -0
  103. package/dist/orchestration/cache.d.ts +15 -0
  104. package/dist/orchestration/cache.js +24 -0
  105. package/dist/orchestration/cache.js.map +1 -0
  106. package/dist/plugin.d.ts +17 -0
  107. package/dist/plugin.js +158 -0
  108. package/dist/plugin.js.map +1 -0
  109. package/dist/read/decoders/deepwiki.d.ts +2 -0
  110. package/dist/read/decoders/deepwiki.js +148 -0
  111. package/dist/read/decoders/deepwiki.js.map +1 -0
  112. package/dist/read/decoders/grokipedia.d.ts +2 -0
  113. package/dist/read/decoders/grokipedia.js +210 -0
  114. package/dist/read/decoders/grokipedia.js.map +1 -0
  115. package/dist/read/decoders/hackernews.d.ts +2 -0
  116. package/dist/read/decoders/hackernews.js +168 -0
  117. package/dist/read/decoders/hackernews.js.map +1 -0
  118. package/dist/read/decoders/index.d.ts +2 -0
  119. package/dist/read/decoders/index.js +12 -0
  120. package/dist/read/decoders/index.js.map +1 -0
  121. package/dist/read/decoders/reddit.d.ts +2 -0
  122. package/dist/read/decoders/reddit.js +142 -0
  123. package/dist/read/decoders/reddit.js.map +1 -0
  124. package/dist/read/decoders/twitter.d.ts +12 -0
  125. package/dist/read/decoders/twitter.js +187 -0
  126. package/dist/read/decoders/twitter.js.map +1 -0
  127. package/dist/read/decoders/wikipedia.d.ts +2 -0
  128. package/dist/read/decoders/wikipedia.js +66 -0
  129. package/dist/read/decoders/wikipedia.js.map +1 -0
  130. package/dist/read/decoders/youtube.d.ts +2 -0
  131. package/dist/read/decoders/youtube.js +69 -0
  132. package/dist/read/decoders/youtube.js.map +1 -0
  133. package/dist/read/extract.d.ts +25 -0
  134. package/dist/read/extract.js +320 -0
  135. package/dist/read/extract.js.map +1 -0
  136. package/dist/read/index.d.ts +14 -0
  137. package/dist/read/index.js +66 -0
  138. package/dist/read/index.js.map +1 -0
  139. package/dist/read/peek.d.ts +9 -0
  140. package/dist/read/peek.js +137 -0
  141. package/dist/read/peek.js.map +1 -0
  142. package/dist/read/types.d.ts +44 -0
  143. package/dist/read/types.js +3 -0
  144. package/dist/read/types.js.map +1 -0
  145. package/dist/replay/engine.d.ts +53 -0
  146. package/dist/replay/engine.js +441 -0
  147. package/dist/replay/engine.js.map +1 -0
  148. package/dist/replay/truncate.d.ts +16 -0
  149. package/dist/replay/truncate.js +92 -0
  150. package/dist/replay/truncate.js.map +1 -0
  151. package/dist/serve.d.ts +31 -0
  152. package/dist/serve.js +149 -0
  153. package/dist/serve.js.map +1 -0
  154. package/dist/skill/generator.d.ts +44 -0
  155. package/dist/skill/generator.js +419 -0
  156. package/dist/skill/generator.js.map +1 -0
  157. package/dist/skill/importer.d.ts +26 -0
  158. package/dist/skill/importer.js +80 -0
  159. package/dist/skill/importer.js.map +1 -0
  160. package/dist/skill/search.d.ts +19 -0
  161. package/dist/skill/search.js +51 -0
  162. package/dist/skill/search.js.map +1 -0
  163. package/dist/skill/signing.d.ts +16 -0
  164. package/dist/skill/signing.js +34 -0
  165. package/dist/skill/signing.js.map +1 -0
  166. package/dist/skill/ssrf.d.ts +27 -0
  167. package/dist/skill/ssrf.js +210 -0
  168. package/dist/skill/ssrf.js.map +1 -0
  169. package/dist/skill/store.d.ts +7 -0
  170. package/dist/skill/store.js +93 -0
  171. package/dist/skill/store.js.map +1 -0
  172. package/dist/stats/report.d.ts +26 -0
  173. package/dist/stats/report.js +157 -0
  174. package/dist/stats/report.js.map +1 -0
  175. package/dist/types.d.ts +214 -0
  176. package/dist/types.js +3 -0
  177. package/dist/types.js.map +1 -0
  178. package/package.json +58 -0
  179. package/src/auth/crypto.ts +92 -0
  180. package/src/auth/handoff.ts +229 -0
  181. package/src/auth/manager.ts +140 -0
  182. package/src/auth/oauth-refresh.ts +120 -0
  183. package/src/auth/refresh.ts +300 -0
  184. package/src/capture/anti-bot.ts +63 -0
  185. package/src/capture/blocklist.ts +75 -0
  186. package/src/capture/body-diff.ts +109 -0
  187. package/src/capture/body-variables.ts +156 -0
  188. package/src/capture/domain.ts +34 -0
  189. package/src/capture/entropy.ts +121 -0
  190. package/src/capture/filter.ts +56 -0
  191. package/src/capture/graphql.ts +124 -0
  192. package/src/capture/idle.ts +45 -0
  193. package/src/capture/monitor.ts +224 -0
  194. package/src/capture/oauth-detector.ts +106 -0
  195. package/src/capture/pagination.ts +49 -0
  196. package/src/capture/parameterize.ts +68 -0
  197. package/src/capture/scrubber.ts +49 -0
  198. package/src/capture/session.ts +502 -0
  199. package/src/capture/token-detector.ts +76 -0
  200. package/src/capture/verifier.ts +171 -0
  201. package/src/cli.ts +1031 -0
  202. package/src/discovery/auth.ts +99 -0
  203. package/src/discovery/fetch.ts +85 -0
  204. package/src/discovery/frameworks.ts +231 -0
  205. package/src/discovery/index.ts +256 -0
  206. package/src/discovery/openapi.ts +230 -0
  207. package/src/discovery/probes.ts +76 -0
  208. package/src/index.ts +26 -0
  209. package/src/inspect/report.ts +247 -0
  210. package/src/mcp.ts +618 -0
  211. package/src/orchestration/browse.ts +250 -0
  212. package/src/orchestration/cache.ts +37 -0
  213. package/src/plugin.ts +188 -0
  214. package/src/read/decoders/deepwiki.ts +180 -0
  215. package/src/read/decoders/grokipedia.ts +246 -0
  216. package/src/read/decoders/hackernews.ts +198 -0
  217. package/src/read/decoders/index.ts +15 -0
  218. package/src/read/decoders/reddit.ts +158 -0
  219. package/src/read/decoders/twitter.ts +211 -0
  220. package/src/read/decoders/wikipedia.ts +75 -0
  221. package/src/read/decoders/youtube.ts +75 -0
  222. package/src/read/extract.ts +396 -0
  223. package/src/read/index.ts +78 -0
  224. package/src/read/peek.ts +175 -0
  225. package/src/read/types.ts +37 -0
  226. package/src/replay/engine.ts +559 -0
  227. package/src/replay/truncate.ts +116 -0
  228. package/src/serve.ts +189 -0
  229. package/src/skill/generator.ts +473 -0
  230. package/src/skill/importer.ts +107 -0
  231. package/src/skill/search.ts +76 -0
  232. package/src/skill/signing.ts +36 -0
  233. package/src/skill/ssrf.ts +238 -0
  234. package/src/skill/store.ts +107 -0
  235. package/src/stats/report.ts +208 -0
  236. package/src/types.ts +233 -0
@@ -0,0 +1,198 @@
1
+ import { readSkillFile } from '../skill/store.js';
2
+ import { replayEndpoint } from '../replay/engine.js';
3
+ import { read } from '../read/index.js';
4
+ /**
5
+ * High-level browse: check cache → disk → discover → replay.
6
+ * Auto-escalates cheap steps. Returns guidance for expensive ones.
7
+ */
8
+ export async function browse(url, options = {}) {
9
+ const { cache, skillsDir, task, skipDiscovery, maxBytes = 50_000 } = options;
10
+ const fullUrl = url.startsWith('http') ? url : `https://${url}`;
11
+ let domain;
12
+ let urlPath;
13
+ try {
14
+ const parsed = new URL(fullUrl);
15
+ domain = parsed.hostname;
16
+ urlPath = parsed.pathname;
17
+ }
18
+ catch {
19
+ return {
20
+ success: false,
21
+ reason: 'invalid_url',
22
+ suggestion: 'provide_valid_url',
23
+ domain: '',
24
+ url: fullUrl,
25
+ task,
26
+ };
27
+ }
28
+ // Step 1: Check session cache
29
+ let skill = null;
30
+ let source = 'disk';
31
+ if (cache?.has(domain)) {
32
+ skill = cache.get(domain).skillFile;
33
+ source = cache.get(domain).source;
34
+ }
35
+ // Step 2: Check disk
36
+ if (!skill) {
37
+ skill = await readSkillFile(domain, skillsDir);
38
+ if (skill) {
39
+ source = 'disk';
40
+ cache?.set(domain, skill, 'disk');
41
+ }
42
+ }
43
+ // Step 3: Try discovery
44
+ if (!skill && !skipDiscovery) {
45
+ try {
46
+ const { discover } = await import('../discovery/index.js');
47
+ const discovery = await discover(fullUrl);
48
+ if (discovery.skillFile && discovery.skillFile.endpoints.length > 0 &&
49
+ (discovery.confidence === 'high' || discovery.confidence === 'medium')) {
50
+ skill = discovery.skillFile;
51
+ source = 'discovered';
52
+ // Save to disk
53
+ const { writeSkillFile: writeSF } = await import('../skill/store.js');
54
+ await writeSF(skill, skillsDir);
55
+ cache?.set(domain, skill, 'discovered');
56
+ }
57
+ else {
58
+ // Discovery didn't produce usable endpoints — try text-mode read
59
+ try {
60
+ const readResult = await read(fullUrl, { maxBytes });
61
+ if (readResult && readResult.content.trim().length > 0 && readResult.metadata.source !== 'spa-shell') {
62
+ return {
63
+ success: true,
64
+ data: readResult,
65
+ status: 200,
66
+ domain,
67
+ endpointId: 'read',
68
+ tier: 'green',
69
+ fromCache: false,
70
+ capturedAt: new Date().toISOString(),
71
+ task,
72
+ };
73
+ }
74
+ }
75
+ catch {
76
+ // Read failed — fall through to capture_needed
77
+ }
78
+ return {
79
+ success: false,
80
+ reason: 'no_replayable_endpoints',
81
+ discoveryConfidence: discovery.confidence,
82
+ suggestion: 'capture_needed',
83
+ domain,
84
+ url: fullUrl,
85
+ task,
86
+ };
87
+ }
88
+ }
89
+ catch {
90
+ // Discovery failed — fall through to guidance
91
+ }
92
+ }
93
+ // No skill file at all — try text-mode read before giving up
94
+ if (!skill) {
95
+ if (!skipDiscovery) {
96
+ try {
97
+ const readResult = await read(fullUrl, { maxBytes });
98
+ if (readResult && readResult.content.trim().length > 0 && readResult.metadata.source !== 'spa-shell') {
99
+ return {
100
+ success: true,
101
+ data: readResult,
102
+ status: 200,
103
+ domain,
104
+ endpointId: 'read',
105
+ tier: 'green',
106
+ fromCache: false,
107
+ capturedAt: new Date().toISOString(),
108
+ task,
109
+ };
110
+ }
111
+ }
112
+ catch {
113
+ // Read failed — fall through to capture_needed
114
+ }
115
+ }
116
+ return {
117
+ success: false,
118
+ reason: 'no_skill_file',
119
+ suggestion: 'capture_needed',
120
+ domain,
121
+ url: fullUrl,
122
+ task,
123
+ };
124
+ }
125
+ // Step 4: Pick best endpoint
126
+ const endpoint = pickEndpoint(skill, urlPath);
127
+ if (!endpoint) {
128
+ return {
129
+ success: false,
130
+ reason: 'no_replayable_endpoints',
131
+ suggestion: 'capture_needed',
132
+ domain,
133
+ url: fullUrl,
134
+ task,
135
+ };
136
+ }
137
+ // Step 5: Replay
138
+ try {
139
+ const result = await replayEndpoint(skill, endpoint.id, { maxBytes, _skipSsrfCheck: options._skipSsrfCheck });
140
+ const fromCache = source === 'disk';
141
+ // Check content-type: HTML responses are not usable API data
142
+ const contentType = result.headers['content-type'] ?? '';
143
+ if (contentType.includes('text/html')) {
144
+ return {
145
+ success: false,
146
+ reason: 'non_api_response',
147
+ discoveryConfidence: source === 'discovered' ? 'medium' : undefined,
148
+ suggestion: 'capture_needed',
149
+ domain,
150
+ url: fullUrl,
151
+ task,
152
+ };
153
+ }
154
+ return {
155
+ success: true,
156
+ data: result.data,
157
+ status: result.status,
158
+ domain,
159
+ endpointId: endpoint.id,
160
+ tier: endpoint.replayability?.tier ?? 'unknown',
161
+ fromCache,
162
+ capturedAt: skill.capturedAt,
163
+ task,
164
+ ...(result.truncated ? { truncated: true } : {}),
165
+ };
166
+ }
167
+ catch {
168
+ return {
169
+ success: false,
170
+ reason: 'replay_failed',
171
+ suggestion: 'capture_needed',
172
+ domain,
173
+ url: fullUrl,
174
+ task,
175
+ };
176
+ }
177
+ }
178
+ const REPLAYABLE_TIERS = new Set(['green', 'yellow', 'unknown']);
179
+ /**
180
+ * Pick the best endpoint to replay. Prefers:
181
+ * 1. GET endpoints with green/yellow/unknown tier
182
+ * 2. Path overlap with the input URL
183
+ * 3. First match as fallback
184
+ */
185
+ function pickEndpoint(skill, urlPath) {
186
+ const candidates = skill.endpoints.filter(ep => ep.method === 'GET' &&
187
+ REPLAYABLE_TIERS.has(ep.replayability?.tier ?? 'unknown'));
188
+ if (candidates.length === 0)
189
+ return null;
190
+ // Prefer path overlap
191
+ if (urlPath && urlPath !== '/') {
192
+ const match = candidates.find(ep => urlPath.includes(ep.path) || ep.path.includes(urlPath));
193
+ if (match)
194
+ return match;
195
+ }
196
+ return candidates[0];
197
+ }
198
+ //# sourceMappingURL=browse.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"browse.js","sourceRoot":"","sources":["../../src/orchestration/browse.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAsCxC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,GAAW,EACX,UAAyB,EAAE;IAE3B,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,GAAG,MAAM,EAAE,GAAG,OAAO,CAAC;IAC7E,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC;IAEhE,IAAI,MAAc,CAAC;IACnB,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC;QACzB,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,UAAU,EAAE,mBAAmB;YAC/B,MAAM,EAAE,EAAE;YACV,GAAG,EAAE,OAAO;YACZ,IAAI;SACL,CAAC;IACJ,CAAC;IAED,8BAA8B;IAC9B,IAAI,KAAK,GAAqB,IAAI,CAAC;IACnC,IAAI,MAAM,GAAuC,MAAM,CAAC;IAExD,IAAI,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QACvB,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,SAAS,CAAC;QACrC,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,GAAG,MAAM,CAAC;YAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,IAAI,CAAC,KAAK,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE1C,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;gBAC/D,CAAC,SAAS,CAAC,UAAU,KAAK,MAAM,IAAI,SAAS,CAAC,UAAU,KAAK,QAAQ,CAAC,EAAE,CAAC;gBAC3E,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC;gBAC5B,MAAM,GAAG,YAAY,CAAC;gBAEtB,eAAe;gBACf,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;gBACtE,MAAM,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAChC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACN,iEAAiE;gBACjE,IAAI,CAAC;oBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACrD,IAAI,UAAU,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;wBACrG,OAAO;4BACL,OAAO,EAAE,IAAI;4BACb,IAAI,EAAE,UAAU;4BAChB,MAAM,EAAE,GAAG;4BACX,MAAM;4BACN,UAAU,EAAE,MAAM;4BAClB,IAAI,EAAE,OAAO;4BACb,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;4BACpC,IAAI;yBACL,CAAC;oBACJ,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,+CAA+C;gBACjD,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,yBAAyB;oBACjC,mBAAmB,EAAE,SAAS,CAAC,UAAU;oBACzC,UAAU,EAAE,gBAAgB;oBAC5B,MAAM;oBACN,GAAG,EAAE,OAAO;oBACZ,IAAI;iBACL,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,8CAA8C;QAChD,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;gBACrD,IAAI,UAAU,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBACrG,OAAO;wBACL,OAAO,EAAE,IAAI;wBACb,IAAI,EAAE,UAAU;wBAChB,MAAM,EAAE,GAAG;wBACX,MAAM;wBACN,UAAU,EAAE,MAAM;wBAClB,IAAI,EAAE,OAAO;wBACb,SAAS,EAAE,KAAK;wBAChB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACpC,IAAI;qBACL,CAAC;gBACJ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,+CAA+C;YACjD,CAAC;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,gBAAgB;YAC5B,MAAM;YACN,GAAG,EAAE,OAAO;YACZ,IAAI;SACL,CAAC;IACJ,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO;YACL,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,yBAAyB;YACjC,UAAU,EAAE,gBAAgB;YAC5B,MAAM;YACN,GAAG,EAAE,OAAO;YACZ,IAAI;SACL,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;QAC9G,MAAM,SAAS,GAAG,MAAM,KAAK,MAAM,CAAC;QAEpC,6DAA6D;QAC7D,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACzD,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,kBAAkB;gBAC1B,mBAAmB,EAAE,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;gBACnE,UAAU,EAAE,gBAAgB;gBAC5B,MAAM;gBACN,GAAG,EAAE,OAAO;gBACZ,IAAI;aACL,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM;YACN,UAAU,EAAE,QAAQ,CAAC,EAAE;YACvB,IAAI,EAAE,QAAQ,CAAC,aAAa,EAAE,IAAI,IAAI,SAAS;YAC/C,SAAS;YACT,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,IAAI;YACJ,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,gBAAgB;YAC5B,MAAM;YACN,GAAG,EAAE,OAAO;YACZ,IAAI;SACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;AAEjE;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAgB,EAAE,OAAe;IACrD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAC7C,EAAE,CAAC,MAAM,KAAK,KAAK;QACnB,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,EAAE,IAAI,IAAI,SAAS,CAAC,CAC1D,CAAC;IAEF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEzC,sBAAsB;IACtB,IAAI,OAAO,IAAI,OAAO,KAAK,GAAG,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5F,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IAED,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}
@@ -0,0 +1,15 @@
1
+ import type { SkillFile } from '../types.js';
2
+ export interface CacheEntry {
3
+ domain: string;
4
+ skillFile: SkillFile;
5
+ discoveredAt: number;
6
+ source: 'disk' | 'discovered' | 'captured';
7
+ }
8
+ export declare class SessionCache {
9
+ private entries;
10
+ set(domain: string, skillFile: SkillFile, source: CacheEntry['source']): void;
11
+ get(domain: string): CacheEntry | null;
12
+ has(domain: string): boolean;
13
+ invalidate(domain: string): void;
14
+ domains(): string[];
15
+ }
@@ -0,0 +1,24 @@
1
+ export class SessionCache {
2
+ entries = new Map();
3
+ set(domain, skillFile, source) {
4
+ this.entries.set(domain, {
5
+ domain,
6
+ skillFile,
7
+ discoveredAt: Date.now(),
8
+ source,
9
+ });
10
+ }
11
+ get(domain) {
12
+ return this.entries.get(domain) ?? null;
13
+ }
14
+ has(domain) {
15
+ return this.entries.has(domain);
16
+ }
17
+ invalidate(domain) {
18
+ this.entries.delete(domain);
19
+ }
20
+ domains() {
21
+ return [...this.entries.keys()];
22
+ }
23
+ }
24
+ //# sourceMappingURL=cache.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/orchestration/cache.ts"],"names":[],"mappings":"AASA,MAAM,OAAO,YAAY;IACf,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAC;IAEhD,GAAG,CAAC,MAAc,EAAE,SAAoB,EAAE,MAA4B;QACpE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;YACvB,MAAM;YACN,SAAS;YACT,YAAY,EAAE,IAAI,CAAC,GAAG,EAAE;YACxB,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAC,MAAc;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAC1C,CAAC;IAED,GAAG,CAAC,MAAc;QAChB,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,UAAU,CAAC,MAAc;QACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,OAAO;QACL,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;CACF"}
@@ -0,0 +1,17 @@
1
+ export interface ToolDefinition {
2
+ name: string;
3
+ description: string;
4
+ parameters: Record<string, unknown>;
5
+ execute: (args: Record<string, unknown>) => Promise<unknown>;
6
+ }
7
+ export interface Plugin {
8
+ name: string;
9
+ version: string;
10
+ tools: ToolDefinition[];
11
+ }
12
+ export interface PluginOptions {
13
+ skillsDir?: string;
14
+ /** @internal Skip SSRF check — for testing only */
15
+ _skipSsrfCheck?: boolean;
16
+ }
17
+ export declare function createPlugin(options?: PluginOptions): Plugin;
package/dist/plugin.js ADDED
@@ -0,0 +1,158 @@
1
+ // src/plugin.ts
2
+ import { searchSkills } from './skill/search.js';
3
+ import { readSkillFile } from './skill/store.js';
4
+ import { replayEndpoint } from './replay/engine.js';
5
+ import { AuthManager, getMachineId } from './auth/manager.js';
6
+ import { homedir } from 'node:os';
7
+ import { join } from 'node:path';
8
+ const APITAP_DIR = join(homedir(), '.apitap');
9
+ export function createPlugin(options = {}) {
10
+ const skillsDir = options.skillsDir;
11
+ const searchTool = {
12
+ name: 'apitap_search',
13
+ description: 'Search available API skill files for a domain or endpoint. ' +
14
+ 'Use this FIRST to check if ApiTap has captured a site\'s API before trying to replay. ' +
15
+ 'Returns matching endpoints with replayability tiers: ' +
16
+ 'green = safe to replay directly, ' +
17
+ 'yellow = needs auth credentials, ' +
18
+ 'orange = fragile (CSRF/session-bound), ' +
19
+ 'red = needs browser (anti-bot). ' +
20
+ 'If not found, use apitap_capture to capture the site first.',
21
+ parameters: {
22
+ type: 'object',
23
+ properties: {
24
+ query: {
25
+ type: 'string',
26
+ description: 'Search query — domain name, endpoint path, or keyword (e.g. "polymarket", "events", "get-markets")',
27
+ },
28
+ },
29
+ required: ['query'],
30
+ },
31
+ execute: async (args) => {
32
+ const query = args.query;
33
+ return searchSkills(query, skillsDir);
34
+ },
35
+ };
36
+ const replayTool = {
37
+ name: 'apitap_replay',
38
+ description: 'Replay a captured API endpoint to get live data. ' +
39
+ 'Check the endpoint tier first with apitap_search: ' +
40
+ 'green = will work, yellow = needs auth, orange/red = may fail. ' +
41
+ 'Returns { status, data } with the API response.',
42
+ parameters: {
43
+ type: 'object',
44
+ properties: {
45
+ domain: {
46
+ type: 'string',
47
+ description: 'Domain of the API (e.g. "gamma-api.polymarket.com")',
48
+ },
49
+ endpointId: {
50
+ type: 'string',
51
+ description: 'Endpoint ID from search results (e.g. "get-events")',
52
+ },
53
+ params: {
54
+ type: 'object',
55
+ description: 'Optional key-value parameters for path substitution or query params (e.g. { "id": "123", "limit": "10" })',
56
+ additionalProperties: { type: 'string' },
57
+ },
58
+ },
59
+ required: ['domain', 'endpointId'],
60
+ },
61
+ execute: async (args) => {
62
+ const domain = args.domain;
63
+ const endpointId = args.endpointId;
64
+ const params = args.params;
65
+ const skill = await readSkillFile(domain, skillsDir);
66
+ if (!skill) {
67
+ return {
68
+ error: `No skill file found for "${domain}". Use apitap_capture to capture it first.`,
69
+ };
70
+ }
71
+ // Inject stored auth if available
72
+ const endpoint = skill.endpoints.find(e => e.id === endpointId);
73
+ if (!endpoint) {
74
+ return {
75
+ error: `Endpoint "${endpointId}" not found. Available: ${skill.endpoints.map(e => e.id).join(', ')}`,
76
+ };
77
+ }
78
+ const hasStoredPlaceholder = Object.values(endpoint.headers).some(v => v === '[stored]');
79
+ if (hasStoredPlaceholder) {
80
+ try {
81
+ const machineId = await getMachineId();
82
+ const authManager = new AuthManager(APITAP_DIR, machineId);
83
+ const storedAuth = await authManager.retrieve(domain);
84
+ if (storedAuth) {
85
+ endpoint.headers[storedAuth.header] = storedAuth.value;
86
+ }
87
+ }
88
+ catch {
89
+ // Auth retrieval failed — proceed without it
90
+ }
91
+ }
92
+ try {
93
+ const result = await replayEndpoint(skill, endpointId, {
94
+ params,
95
+ _skipSsrfCheck: options._skipSsrfCheck,
96
+ });
97
+ return { status: result.status, data: result.data };
98
+ }
99
+ catch (err) {
100
+ return { error: err.message };
101
+ }
102
+ },
103
+ };
104
+ const captureTool = {
105
+ name: 'apitap_capture',
106
+ description: 'Capture a website\'s API traffic by browsing it with an instrumented browser. ' +
107
+ 'Use this when apitap_search returns no results for a site. ' +
108
+ 'Launches a browser, navigates to the URL, captures API calls for the specified duration, ' +
109
+ 'and generates a skill file for future replay. ' +
110
+ 'Returns { domains, endpoints, skillFiles } summary.',
111
+ parameters: {
112
+ type: 'object',
113
+ properties: {
114
+ url: {
115
+ type: 'string',
116
+ description: 'URL to capture (e.g. "https://polymarket.com")',
117
+ },
118
+ duration: {
119
+ type: 'number',
120
+ description: 'Capture duration in seconds (default: 30)',
121
+ },
122
+ allDomains: {
123
+ type: 'boolean',
124
+ description: 'Capture all domains, not just the target domain (default: false)',
125
+ },
126
+ },
127
+ required: ['url'],
128
+ },
129
+ execute: async (args) => {
130
+ const url = args.url;
131
+ const duration = args.duration ?? 30;
132
+ const allDomains = args.allDomains ?? false;
133
+ // Shell out to CLI for capture (it handles browser lifecycle, signing, etc.)
134
+ const { execFile } = await import('node:child_process');
135
+ const { promisify } = await import('node:util');
136
+ const execFileAsync = promisify(execFile);
137
+ const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json', '--no-verify'];
138
+ if (allDomains)
139
+ cliArgs.push('--all-domains');
140
+ try {
141
+ const { stdout } = await execFileAsync('node', cliArgs, {
142
+ timeout: (duration + 30) * 1000,
143
+ env: { ...process.env, ...(skillsDir ? { APITAP_SKILLS_DIR: skillsDir } : {}) },
144
+ });
145
+ return JSON.parse(stdout);
146
+ }
147
+ catch (err) {
148
+ return { error: `Capture failed: ${err.message}` };
149
+ }
150
+ },
151
+ };
152
+ return {
153
+ name: 'apitap',
154
+ version: '0.4.0',
155
+ tools: [searchTool, replayTool, captureTool],
156
+ };
157
+ }
158
+ //# sourceMappingURL=plugin.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"plugin.js","sourceRoot":"","sources":["../src/plugin.ts"],"names":[],"mappings":"AAAA,gBAAgB;AAChB,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAqBjC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;AAE9C,MAAM,UAAU,YAAY,CAAC,UAAyB,EAAE;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAEpC,MAAM,UAAU,GAAmB;QACjC,IAAI,EAAE,eAAe;QACrB,WAAW,EACT,6DAA6D;YAC7D,wFAAwF;YACxF,uDAAuD;YACvD,mCAAmC;YACnC,mCAAmC;YACnC,yCAAyC;YACzC,kCAAkC;YAClC,6DAA6D;QAC/D,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,oGAAoG;iBAClH;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAe,CAAC;YACnC,OAAO,YAAY,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACxC,CAAC;KACF,CAAC;IAEF,MAAM,UAAU,GAAmB;QACjC,IAAI,EAAE,eAAe;QACrB,WAAW,EACT,mDAAmD;YACnD,oDAAoD;YACpD,iEAAiE;YACjE,iDAAiD;QACnD,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qDAAqD;iBACnE;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qDAAqD;iBACnE;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,2GAA2G;oBACxH,oBAAoB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;iBACzC;aACF;YACD,QAAQ,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC;SACnC;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAgB,CAAC;YACrC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAoB,CAAC;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,MAA4C,CAAC;YAEjE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;oBACL,KAAK,EAAE,4BAA4B,MAAM,4CAA4C;iBACtF,CAAC;YACJ,CAAC;YAED,kCAAkC;YAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,UAAU,CAAC,CAAC;YAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;oBACL,KAAK,EAAE,aAAa,UAAU,2BAA2B,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACrG,CAAC;YACJ,CAAC;YAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC;YACzF,IAAI,oBAAoB,EAAE,CAAC;gBACzB,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;oBACvC,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC3D,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACtD,IAAI,UAAU,EAAE,CAAC;wBACf,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC;oBACzD,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,6CAA6C;gBAC/C,CAAC;YACH,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,UAAU,EAAE;oBACrD,MAAM;oBACN,cAAc,EAAE,OAAO,CAAC,cAAc;iBACvC,CAAC,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;QACH,CAAC;KACF,CAAC;IAEF,MAAM,WAAW,GAAmB;QAClC,IAAI,EAAE,gBAAgB;QACtB,WAAW,EACT,gFAAgF;YAChF,6DAA6D;YAC7D,2FAA2F;YAC3F,gDAAgD;YAChD,qDAAqD;QACvD,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,gDAAgD;iBAC9D;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,2CAA2C;iBACzD;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,kEAAkE;iBAChF;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,CAAC;SAClB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAa,CAAC;YAC/B,MAAM,QAAQ,GAAI,IAAI,CAAC,QAAmB,IAAI,EAAE,CAAC;YACjD,MAAM,UAAU,GAAI,IAAI,CAAC,UAAsB,IAAI,KAAK,CAAC;YAEzD,6EAA6E;YAC7E,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACxD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;YAChD,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;YAE1C,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;YAC3H,IAAI,UAAU;gBAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAE9C,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE;oBACtD,OAAO,EAAE,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,IAAI;oBAC/B,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;iBAChF,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,OAAO,EAAE,KAAK,EAAE,mBAAmB,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,CAAC;QACH,CAAC;KACF,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,OAAO;QAChB,KAAK,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,CAAC;KAC7C,CAAC;AACJ,CAAC"}
@@ -0,0 +1,2 @@
1
+ import type { Decoder } from '../types.js';
2
+ export declare const deepwikiDecoder: Decoder;
@@ -0,0 +1,148 @@
1
+ function estimateTokens(text) {
2
+ return Math.ceil(text.length / 4);
3
+ }
4
+ /**
5
+ * DeepWiki decoder — extracts wiki content from deepwiki.com
6
+ *
7
+ * DeepWiki (by Devin/Cognition) auto-generates documentation wikis from GitHub repos.
8
+ * It's a Next.js app that serves content via React Server Components (RSC).
9
+ *
10
+ * Trick: Send `RSC: 1` header → get full markdown content in the RSC payload
11
+ * instead of the JS-heavy SPA shell. No auth required.
12
+ *
13
+ * URL patterns:
14
+ * deepwiki.com/{org}/{repo} → overview page
15
+ * deepwiki.com/{org}/{repo}/{page-slug} → specific wiki page
16
+ */
17
+ const DEEPWIKI_PATTERN = /^https?:\/\/(www\.)?deepwiki\.com\/([^/]+)\/([^/]+)(\/.*)?$/;
18
+ export const deepwikiDecoder = {
19
+ name: 'deepwiki',
20
+ patterns: [
21
+ /^https?:\/\/(www\.)?deepwiki\.com\/[^/]+\/[^/]+/,
22
+ ],
23
+ async decode(url, options = {}) {
24
+ const match = url.match(DEEPWIKI_PATTERN);
25
+ if (!match)
26
+ return null;
27
+ const org = match[2];
28
+ const repo = match[3];
29
+ const pagePath = match[4] || '';
30
+ // Construct the path for the RSC request
31
+ const fullPath = `/${org}/${repo}${pagePath}`;
32
+ try {
33
+ const response = await fetch(url, {
34
+ headers: {
35
+ 'RSC': '1',
36
+ 'Next-Url': fullPath,
37
+ 'User-Agent': 'Mozilla/5.0 (compatible; ApiTap/1.0)',
38
+ },
39
+ signal: AbortSignal.timeout(10_000),
40
+ });
41
+ if (!response.ok) {
42
+ return null;
43
+ }
44
+ const rscPayload = await response.text();
45
+ // Extract markdown content from RSC text nodes
46
+ // Format: {id}:T{hexLength},{content}
47
+ const contentBlocks = [];
48
+ const lines = rscPayload.split('\n');
49
+ let currentBlock = null;
50
+ let expectedLength = 0;
51
+ let collectedBytes = 0;
52
+ for (const line of lines) {
53
+ // Start of a new text block: {id}:T{hexLength},{content...}
54
+ const blockMatch = line.match(/^[0-9a-f]+:T([0-9a-f]+),(.*)$/);
55
+ if (blockMatch) {
56
+ // Save previous block if exists
57
+ if (currentBlock !== null) {
58
+ contentBlocks.push(currentBlock);
59
+ }
60
+ expectedLength = parseInt(blockMatch[1], 16);
61
+ const content = blockMatch[2];
62
+ currentBlock = content;
63
+ collectedBytes = Buffer.byteLength(content, 'utf-8');
64
+ continue;
65
+ }
66
+ // If we're inside a block, keep collecting lines
67
+ if (currentBlock !== null) {
68
+ // Check if this line starts a new RSC record (not a continuation)
69
+ if (/^[0-9a-f]+:[^T]/.test(line) || /^[0-9a-f]+:T[0-9a-f]+,/.test(line)) {
70
+ // End of current block
71
+ contentBlocks.push(currentBlock);
72
+ currentBlock = null;
73
+ // If it's a new T block, process it
74
+ const newBlock = line.match(/^[0-9a-f]+:T([0-9a-f]+),(.*)$/);
75
+ if (newBlock) {
76
+ expectedLength = parseInt(newBlock[1], 16);
77
+ currentBlock = newBlock[2];
78
+ collectedBytes = Buffer.byteLength(newBlock[2], 'utf-8');
79
+ }
80
+ continue;
81
+ }
82
+ currentBlock += '\n' + line;
83
+ collectedBytes += Buffer.byteLength('\n' + line, 'utf-8');
84
+ // If we've collected enough bytes, end the block
85
+ if (collectedBytes >= expectedLength) {
86
+ contentBlocks.push(currentBlock);
87
+ currentBlock = null;
88
+ }
89
+ }
90
+ }
91
+ // Don't forget the last block
92
+ if (currentBlock !== null) {
93
+ contentBlocks.push(currentBlock);
94
+ }
95
+ if (contentBlocks.length === 0) {
96
+ return null;
97
+ }
98
+ // Find the largest content block — that's the main page content
99
+ // (smaller blocks might be TOC section titles)
100
+ const mainContent = contentBlocks.reduce((a, b) => a.length > b.length ? a : b);
101
+ if (!mainContent || mainContent.length < 50) {
102
+ return null;
103
+ }
104
+ // Clean up the markdown
105
+ let content = mainContent;
106
+ // Extract title from first heading
107
+ const titleMatch = content.match(/^#\s+(.+)$/m);
108
+ const title = titleMatch
109
+ ? `${titleMatch[1]} — ${org}/${repo} | DeepWiki`
110
+ : `${org}/${repo} | DeepWiki`;
111
+ // Fix relative links to point back to correct locations
112
+ content = content.replace(/\[([^\]]+)\]\((?!https?:\/\/)([^)]+)\)/g, (full, text, href) => {
113
+ // Source file links (e.g., README.md, src/foo.ts)
114
+ if (href.match(/\.(ts|js|md|json|tsx|jsx|py|rs|go|toml|yaml|yml|css|html)$/)) {
115
+ return `[${text}](https://github.com/${org}/${repo}/blob/main/${href})`;
116
+ }
117
+ // Section links (#2, #3.1, etc.)
118
+ if (href.startsWith('#')) {
119
+ return `[${text}](https://deepwiki.com/${org}/${repo}/${href.slice(1)})`;
120
+ }
121
+ // Other relative links
122
+ return `[${text}](https://deepwiki.com/${org}/${repo}/${href})`;
123
+ });
124
+ const tokens = estimateTokens(content);
125
+ return {
126
+ url,
127
+ title,
128
+ author: null,
129
+ description: `DeepWiki documentation for ${org}/${repo}`,
130
+ content,
131
+ links: [],
132
+ images: [],
133
+ metadata: {
134
+ type: 'wiki',
135
+ publishedAt: null,
136
+ source: 'deepwiki-rsc',
137
+ canonical: `https://deepwiki.com/${org}/${repo}${pagePath}`,
138
+ siteName: 'DeepWiki',
139
+ },
140
+ cost: { tokens },
141
+ };
142
+ }
143
+ catch {
144
+ return null;
145
+ }
146
+ },
147
+ };
148
+ //# sourceMappingURL=deepwiki.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deepwiki.js","sourceRoot":"","sources":["../../../src/read/decoders/deepwiki.ts"],"names":[],"mappings":"AAGA,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;;GAYG;AAEH,MAAM,gBAAgB,GAAG,6DAA6D,CAAC;AAEvF,MAAM,CAAC,MAAM,eAAe,GAAY;IACtC,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE;QACR,iDAAiD;KAClD;IAED,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,UAAsD,EAAE;QAChF,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAEhC,yCAAyC;QACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;QAE9C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,OAAO,EAAE;oBACP,KAAK,EAAE,GAAG;oBACV,UAAU,EAAE,QAAQ;oBACpB,YAAY,EAAE,sCAAsC;iBACrD;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEzC,+CAA+C;YAC/C,sCAAsC;YACtC,MAAM,aAAa,GAAa,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAErC,IAAI,YAAY,GAAkB,IAAI,CAAC;YACvC,IAAI,cAAc,GAAG,CAAC,CAAC;YACvB,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,4DAA4D;gBAC5D,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAE/D,IAAI,UAAU,EAAE,CAAC;oBACf,gCAAgC;oBAChC,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;wBAC1B,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBACnC,CAAC;oBAED,cAAc,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC7C,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;oBAC9B,YAAY,GAAG,OAAO,CAAC;oBACvB,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,iDAAiD;gBACjD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;oBAC1B,kEAAkE;oBAClE,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;wBACxE,uBAAuB;wBACvB,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBACjC,YAAY,GAAG,IAAI,CAAC;wBAEpB,oCAAoC;wBACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;wBAC7D,IAAI,QAAQ,EAAE,CAAC;4BACb,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;4BAC3C,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;4BAC3B,cAAc,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;wBAC3D,CAAC;wBACD,SAAS;oBACX,CAAC;oBAED,YAAY,IAAI,IAAI,GAAG,IAAI,CAAC;oBAC5B,cAAc,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;oBAE1D,iDAAiD;oBACjD,IAAI,cAAc,IAAI,cAAc,EAAE,CAAC;wBACrC,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBACjC,YAAY,GAAG,IAAI,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;gBAC1B,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACnC,CAAC;YAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/B,OAAO,IAAI,CAAC;YACd,CAAC;YAED,gEAAgE;YAChE,+CAA+C;YAC/C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAChD,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC5B,CAAC;YAEF,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBAC5C,OAAO,IAAI,CAAC;YACd,CAAC;YAED,wBAAwB;YACxB,IAAI,OAAO,GAAG,WAAW,CAAC;YAE1B,mCAAmC;YACnC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,UAAU;gBACtB,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,IAAI,aAAa;gBAChD,CAAC,CAAC,GAAG,GAAG,IAAI,IAAI,aAAa,CAAC;YAEhC,wDAAwD;YACxD,OAAO,GAAG,OAAO,CAAC,OAAO,CACvB,yCAAyC,EACzC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;gBACnB,kDAAkD;gBAClD,IAAI,IAAI,CAAC,KAAK,CAAC,4DAA4D,CAAC,EAAE,CAAC;oBAC7E,OAAO,IAAI,IAAI,wBAAwB,GAAG,IAAI,IAAI,cAAc,IAAI,GAAG,CAAC;gBAC1E,CAAC;gBACD,iCAAiC;gBACjC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,OAAO,IAAI,IAAI,0BAA0B,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;gBAC3E,CAAC;gBACD,uBAAuB;gBACvB,OAAO,IAAI,IAAI,0BAA0B,GAAG,IAAI,IAAI,IAAI,IAAI,GAAG,CAAC;YAClE,CAAC,CACF,CAAC;YAEF,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;YAEvC,OAAO;gBACL,GAAG;gBACH,KAAK;gBACL,MAAM,EAAE,IAAI;gBACZ,WAAW,EAAE,8BAA8B,GAAG,IAAI,IAAI,EAAE;gBACxD,OAAO;gBACP,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,MAAM;oBACZ,WAAW,EAAE,IAAI;oBACjB,MAAM,EAAE,cAAc;oBACtB,SAAS,EAAE,wBAAwB,GAAG,IAAI,IAAI,GAAG,QAAQ,EAAE;oBAC3D,QAAQ,EAAE,UAAU;iBACrB;gBACD,IAAI,EAAE,EAAE,MAAM,EAAE;aACjB,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF,CAAC"}
@@ -0,0 +1,2 @@
1
+ import type { Decoder } from '../types.js';
2
+ export declare const grokipediaDecoder: Decoder;