@sholajegede/bright-data-sync 0.1.1-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +182 -0
- package/dist/client/_generated/_ignore.d.ts +1 -0
- package/dist/client/_generated/_ignore.d.ts.map +1 -0
- package/dist/client/_generated/_ignore.js +3 -0
- package/dist/client/_generated/_ignore.js.map +1 -0
- package/dist/client/index.d.ts +77 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +61 -0
- package/dist/client/index.js.map +1 -0
- package/dist/component/_generated/api.d.ts +34 -0
- package/dist/component/_generated/api.d.ts.map +1 -0
- package/dist/component/_generated/api.js +31 -0
- package/dist/component/_generated/api.js.map +1 -0
- package/dist/component/_generated/component.d.ts +67 -0
- package/dist/component/_generated/component.d.ts.map +1 -0
- package/dist/component/_generated/component.js +11 -0
- package/dist/component/_generated/component.js.map +1 -0
- package/dist/component/_generated/dataModel.d.ts +46 -0
- package/dist/component/_generated/dataModel.d.ts.map +1 -0
- package/dist/component/_generated/dataModel.js +11 -0
- package/dist/component/_generated/dataModel.js.map +1 -0
- package/dist/component/_generated/server.d.ts +121 -0
- package/dist/component/_generated/server.d.ts.map +1 -0
- package/dist/component/_generated/server.js +78 -0
- package/dist/component/_generated/server.js.map +1 -0
- package/dist/component/convex.config.d.ts +3 -0
- package/dist/component/convex.config.d.ts.map +1 -0
- package/dist/component/convex.config.js +3 -0
- package/dist/component/convex.config.js.map +1 -0
- package/dist/component/lib.d.ts +83 -0
- package/dist/component/lib.d.ts.map +1 -0
- package/dist/component/lib.js +314 -0
- package/dist/component/lib.js.map +1 -0
- package/dist/component/schema.d.ts +47 -0
- package/dist/component/schema.d.ts.map +1 -0
- package/dist/component/schema.js +28 -0
- package/dist/component/schema.js.map +1 -0
- package/dist/react/index.d.ts +2 -0
- package/dist/react/index.d.ts.map +1 -0
- package/dist/react/index.js +6 -0
- package/dist/react/index.js.map +1 -0
- package/package.json +102 -0
- package/src/client/_generated/_ignore.ts +1 -0
- package/src/client/index.test.ts +13 -0
- package/src/client/index.ts +100 -0
- package/src/client/setup.test.ts +26 -0
- package/src/component/_generated/api.ts +50 -0
- package/src/component/_generated/component.ts +85 -0
- package/src/component/_generated/dataModel.ts +60 -0
- package/src/component/_generated/server.ts +156 -0
- package/src/component/convex.config.ts +3 -0
- package/src/component/lib.test.ts +21 -0
- package/src/component/lib.ts +375 -0
- package/src/component/schema.ts +30 -0
- package/src/component/setup.test.ts +11 -0
- package/src/react/index.ts +7 -0
- package/src/test.ts +18 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
import { v } from "convex/values";
|
|
2
|
+
import { action, internalMutation, internalQuery, query, } from "./_generated/server.js";
|
|
3
|
+
import { internal } from "./_generated/api.js";
|
|
4
|
+
// ─── Constants ───────────────────────────────────────────────────────────────
|
|
5
|
+
const DEFAULT_SEARCH_TTL_MS = 1000 * 60 * 60; // 1 hour
|
|
6
|
+
const DEFAULT_PAGE_TTL_MS = 1000 * 60 * 60 * 6; // 6 hours
|
|
7
|
+
// ─── Internal helpers ────────────────────────────────────────────────────────
|
|
8
|
+
export const getSearchByQuery = internalQuery({
|
|
9
|
+
args: { query: v.string() },
|
|
10
|
+
returns: v.union(v.null(), v.object({
|
|
11
|
+
_id: v.id("searches"),
|
|
12
|
+
_creationTime: v.number(),
|
|
13
|
+
query: v.string(),
|
|
14
|
+
vertical: v.optional(v.string()),
|
|
15
|
+
recency: v.optional(v.string()),
|
|
16
|
+
results: v.string(),
|
|
17
|
+
fetchedAt: v.number(),
|
|
18
|
+
expiresAt: v.number(),
|
|
19
|
+
})),
|
|
20
|
+
handler: async (ctx, args) => {
|
|
21
|
+
return await ctx.db
|
|
22
|
+
.query("searches")
|
|
23
|
+
.withIndex("by_query", (q) => q.eq("query", args.query))
|
|
24
|
+
.first();
|
|
25
|
+
},
|
|
26
|
+
});
|
|
27
|
+
export const getPageByUrl = internalQuery({
|
|
28
|
+
args: { url: v.string() },
|
|
29
|
+
returns: v.union(v.null(), v.object({
|
|
30
|
+
_id: v.id("pages"),
|
|
31
|
+
_creationTime: v.number(),
|
|
32
|
+
url: v.string(),
|
|
33
|
+
content: v.string(),
|
|
34
|
+
fetchedAt: v.number(),
|
|
35
|
+
expiresAt: v.number(),
|
|
36
|
+
})),
|
|
37
|
+
handler: async (ctx, args) => {
|
|
38
|
+
return await ctx.db
|
|
39
|
+
.query("pages")
|
|
40
|
+
.withIndex("by_url", (q) => q.eq("url", args.url))
|
|
41
|
+
.first();
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
export const upsertSearch = internalMutation({
|
|
45
|
+
args: {
|
|
46
|
+
query: v.string(),
|
|
47
|
+
vertical: v.optional(v.string()),
|
|
48
|
+
recency: v.optional(v.string()),
|
|
49
|
+
results: v.string(),
|
|
50
|
+
ttlMs: v.optional(v.number()),
|
|
51
|
+
},
|
|
52
|
+
returns: v.id("searches"),
|
|
53
|
+
handler: async (ctx, args) => {
|
|
54
|
+
const now = Date.now();
|
|
55
|
+
const expiresAt = now + (args.ttlMs ?? DEFAULT_SEARCH_TTL_MS);
|
|
56
|
+
const existing = await ctx.db
|
|
57
|
+
.query("searches")
|
|
58
|
+
.withIndex("by_query", (q) => q.eq("query", args.query))
|
|
59
|
+
.first();
|
|
60
|
+
if (existing) {
|
|
61
|
+
await ctx.db.patch("searches", existing._id, {
|
|
62
|
+
results: args.results,
|
|
63
|
+
fetchedAt: now,
|
|
64
|
+
expiresAt,
|
|
65
|
+
vertical: args.vertical,
|
|
66
|
+
recency: args.recency,
|
|
67
|
+
});
|
|
68
|
+
return existing._id;
|
|
69
|
+
}
|
|
70
|
+
return await ctx.db.insert("searches", {
|
|
71
|
+
query: args.query,
|
|
72
|
+
vertical: args.vertical,
|
|
73
|
+
recency: args.recency,
|
|
74
|
+
results: args.results,
|
|
75
|
+
fetchedAt: now,
|
|
76
|
+
expiresAt,
|
|
77
|
+
});
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
export const upsertPage = internalMutation({
|
|
81
|
+
args: {
|
|
82
|
+
url: v.string(),
|
|
83
|
+
content: v.string(),
|
|
84
|
+
ttlMs: v.optional(v.number()),
|
|
85
|
+
},
|
|
86
|
+
returns: v.id("pages"),
|
|
87
|
+
handler: async (ctx, args) => {
|
|
88
|
+
const now = Date.now();
|
|
89
|
+
const expiresAt = now + (args.ttlMs ?? DEFAULT_PAGE_TTL_MS);
|
|
90
|
+
const existing = await ctx.db
|
|
91
|
+
.query("pages")
|
|
92
|
+
.withIndex("by_url", (q) => q.eq("url", args.url))
|
|
93
|
+
.first();
|
|
94
|
+
if (existing) {
|
|
95
|
+
await ctx.db.patch("pages", existing._id, {
|
|
96
|
+
content: args.content,
|
|
97
|
+
fetchedAt: now,
|
|
98
|
+
expiresAt,
|
|
99
|
+
});
|
|
100
|
+
return existing._id;
|
|
101
|
+
}
|
|
102
|
+
return await ctx.db.insert("pages", {
|
|
103
|
+
url: args.url,
|
|
104
|
+
content: args.content,
|
|
105
|
+
fetchedAt: now,
|
|
106
|
+
expiresAt,
|
|
107
|
+
});
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
export const deleteSearchByQuery = internalMutation({
|
|
111
|
+
args: { query: v.string() },
|
|
112
|
+
returns: v.null(),
|
|
113
|
+
handler: async (ctx, args) => {
|
|
114
|
+
const existing = await ctx.db
|
|
115
|
+
.query("searches")
|
|
116
|
+
.withIndex("by_query", (q) => q.eq("query", args.query))
|
|
117
|
+
.first();
|
|
118
|
+
if (existing)
|
|
119
|
+
await ctx.db.delete("searches", existing._id);
|
|
120
|
+
return null;
|
|
121
|
+
},
|
|
122
|
+
});
|
|
123
|
+
export const deletePageByUrl = internalMutation({
|
|
124
|
+
args: { url: v.string() },
|
|
125
|
+
returns: v.null(),
|
|
126
|
+
handler: async (ctx, args) => {
|
|
127
|
+
const existing = await ctx.db
|
|
128
|
+
.query("pages")
|
|
129
|
+
.withIndex("by_url", (q) => q.eq("url", args.url))
|
|
130
|
+
.first();
|
|
131
|
+
if (existing)
|
|
132
|
+
await ctx.db.delete("pages", existing._id);
|
|
133
|
+
return null;
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
// ─── Public queries (reactive) ───────────────────────────────────────────────
|
|
137
|
+
export const getSearch = query({
|
|
138
|
+
args: { query: v.string() },
|
|
139
|
+
returns: v.union(v.null(), v.object({
|
|
140
|
+
results: v.string(),
|
|
141
|
+
fetchedAt: v.number(),
|
|
142
|
+
expiresAt: v.number(),
|
|
143
|
+
isFresh: v.boolean(),
|
|
144
|
+
})),
|
|
145
|
+
handler: async (ctx, args) => {
|
|
146
|
+
const row = await ctx.db
|
|
147
|
+
.query("searches")
|
|
148
|
+
.withIndex("by_query", (q) => q.eq("query", args.query))
|
|
149
|
+
.first();
|
|
150
|
+
if (!row)
|
|
151
|
+
return null;
|
|
152
|
+
return {
|
|
153
|
+
results: row.results,
|
|
154
|
+
fetchedAt: row.fetchedAt,
|
|
155
|
+
expiresAt: row.expiresAt,
|
|
156
|
+
isFresh: Date.now() < row.expiresAt,
|
|
157
|
+
};
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
export const getPage = query({
|
|
161
|
+
args: { url: v.string() },
|
|
162
|
+
returns: v.union(v.null(), v.object({
|
|
163
|
+
content: v.string(),
|
|
164
|
+
fetchedAt: v.number(),
|
|
165
|
+
expiresAt: v.number(),
|
|
166
|
+
isFresh: v.boolean(),
|
|
167
|
+
})),
|
|
168
|
+
handler: async (ctx, args) => {
|
|
169
|
+
const row = await ctx.db
|
|
170
|
+
.query("pages")
|
|
171
|
+
.withIndex("by_url", (q) => q.eq("url", args.url))
|
|
172
|
+
.first();
|
|
173
|
+
if (!row)
|
|
174
|
+
return null;
|
|
175
|
+
return {
|
|
176
|
+
content: row.content,
|
|
177
|
+
fetchedAt: row.fetchedAt,
|
|
178
|
+
expiresAt: row.expiresAt,
|
|
179
|
+
isFresh: Date.now() < row.expiresAt,
|
|
180
|
+
};
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
// ─── Public actions ──────────────────────────────────────────────────────────
|
|
184
|
+
export const search = action({
|
|
185
|
+
args: {
|
|
186
|
+
query: v.string(),
|
|
187
|
+
vertical: v.optional(v.string()),
|
|
188
|
+
recency: v.optional(v.string()),
|
|
189
|
+
ttlMs: v.optional(v.number()),
|
|
190
|
+
// Passed in from client wrapper — never stored
|
|
191
|
+
brightdataApiToken: v.string(),
|
|
192
|
+
brightdataSearchZone: v.optional(v.string()),
|
|
193
|
+
},
|
|
194
|
+
returns: v.object({
|
|
195
|
+
results: v.string(),
|
|
196
|
+
fromCache: v.boolean(),
|
|
197
|
+
fetchedAt: v.number(),
|
|
198
|
+
}),
|
|
199
|
+
handler: async (ctx, args) => {
|
|
200
|
+
// 1. Check cache
|
|
201
|
+
// 1. Check cache
|
|
202
|
+
const cached = (await ctx.runQuery(internal.lib.getSearchByQuery, {
|
|
203
|
+
query: args.query,
|
|
204
|
+
}));
|
|
205
|
+
if (cached && Date.now() < cached.expiresAt) {
|
|
206
|
+
return {
|
|
207
|
+
results: cached.results,
|
|
208
|
+
fromCache: true,
|
|
209
|
+
fetchedAt: cached.fetchedAt,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
// 2. Fetch from Bright Data SERP API
|
|
213
|
+
const zone = args.brightdataSearchZone ?? "serp_api1";
|
|
214
|
+
const url = new URL("https://api.brightdata.com/request");
|
|
215
|
+
const body = {
|
|
216
|
+
zone,
|
|
217
|
+
url: `https://www.google.com/search?q=${encodeURIComponent(args.query)}${args.vertical ? `&tbm=${args.vertical}` : ""}${args.recency ? `&tbs=qdr:${args.recency}` : ""}`,
|
|
218
|
+
format: "json",
|
|
219
|
+
};
|
|
220
|
+
const response = await fetch(url.toString(), {
|
|
221
|
+
method: "POST",
|
|
222
|
+
headers: {
|
|
223
|
+
"Content-Type": "application/json",
|
|
224
|
+
Authorization: `Bearer ${args.brightdataApiToken}`,
|
|
225
|
+
},
|
|
226
|
+
body: JSON.stringify(body),
|
|
227
|
+
});
|
|
228
|
+
if (!response.ok) {
|
|
229
|
+
throw new Error(`Bright Data SERP API error: ${response.status} ${await response.text()}`);
|
|
230
|
+
}
|
|
231
|
+
const results = await response.text();
|
|
232
|
+
const now = Date.now();
|
|
233
|
+
// 3. Store in cache
|
|
234
|
+
await ctx.runMutation(internal.lib.upsertSearch, {
|
|
235
|
+
query: args.query,
|
|
236
|
+
vertical: args.vertical,
|
|
237
|
+
recency: args.recency,
|
|
238
|
+
results,
|
|
239
|
+
ttlMs: args.ttlMs,
|
|
240
|
+
});
|
|
241
|
+
return { results, fromCache: false, fetchedAt: now };
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
export const scrape = action({
|
|
245
|
+
args: {
|
|
246
|
+
url: v.string(),
|
|
247
|
+
ttlMs: v.optional(v.number()),
|
|
248
|
+
brightdataApiToken: v.string(),
|
|
249
|
+
brightdataWebUnlockerZone: v.optional(v.string()),
|
|
250
|
+
},
|
|
251
|
+
returns: v.object({
|
|
252
|
+
content: v.string(),
|
|
253
|
+
fromCache: v.boolean(),
|
|
254
|
+
fetchedAt: v.number(),
|
|
255
|
+
}),
|
|
256
|
+
handler: async (ctx, args) => {
|
|
257
|
+
// 1. Check cache
|
|
258
|
+
const cached = (await ctx.runQuery(internal.lib.getPageByUrl, {
|
|
259
|
+
url: args.url,
|
|
260
|
+
}));
|
|
261
|
+
if (cached && Date.now() < cached.expiresAt) {
|
|
262
|
+
return {
|
|
263
|
+
content: cached.content,
|
|
264
|
+
fromCache: true,
|
|
265
|
+
fetchedAt: cached.fetchedAt,
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
// 2. Fetch from Bright Data Web Unlocker
|
|
269
|
+
const zone = args.brightdataWebUnlockerZone ?? "web_unlocker1";
|
|
270
|
+
const response = await fetch("https://api.brightdata.com/request", {
|
|
271
|
+
method: "POST",
|
|
272
|
+
headers: {
|
|
273
|
+
"Content-Type": "application/json",
|
|
274
|
+
Authorization: `Bearer ${args.brightdataApiToken}`,
|
|
275
|
+
},
|
|
276
|
+
body: JSON.stringify({
|
|
277
|
+
zone,
|
|
278
|
+
url: args.url,
|
|
279
|
+
format: "raw",
|
|
280
|
+
}),
|
|
281
|
+
});
|
|
282
|
+
if (!response.ok) {
|
|
283
|
+
throw new Error(`Bright Data Web Unlocker error: ${response.status} ${await response.text()}`);
|
|
284
|
+
}
|
|
285
|
+
const content = await response.text();
|
|
286
|
+
const now = Date.now();
|
|
287
|
+
// 3. Store in cache
|
|
288
|
+
await ctx.runMutation(internal.lib.upsertPage, {
|
|
289
|
+
url: args.url,
|
|
290
|
+
content,
|
|
291
|
+
ttlMs: args.ttlMs,
|
|
292
|
+
});
|
|
293
|
+
return { content, fromCache: false, fetchedAt: now };
|
|
294
|
+
},
|
|
295
|
+
});
|
|
296
|
+
export const invalidate = action({
|
|
297
|
+
args: {
|
|
298
|
+
query: v.optional(v.string()),
|
|
299
|
+
url: v.optional(v.string()),
|
|
300
|
+
},
|
|
301
|
+
returns: v.null(),
|
|
302
|
+
handler: async (ctx, args) => {
|
|
303
|
+
if (args.query) {
|
|
304
|
+
await ctx.runMutation(internal.lib.deleteSearchByQuery, {
|
|
305
|
+
query: args.query,
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
if (args.url) {
|
|
309
|
+
await ctx.runMutation(internal.lib.deletePageByUrl, { url: args.url });
|
|
310
|
+
}
|
|
311
|
+
return null;
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
//# sourceMappingURL=lib.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lib.js","sourceRoot":"","sources":["../../src/component/lib.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EACL,MAAM,EACN,gBAAgB,EAChB,aAAa,EACb,KAAK,GACN,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,gFAAgF;AAEhF,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,SAAS;AACvD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU;AAE1D,gFAAgF;AAEhF,MAAM,CAAC,MAAM,gBAAgB,GAAG,aAAa,CAAC;IAC5C,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,IAAI,EAAE,EACR,CAAC,CAAC,MAAM,CAAC;QACP,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC;QACrB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC,CACH;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,OAAO,MAAM,GAAG,CAAC,EAAE;aAChB,KAAK,CAAC,UAAU,CAAC;aACjB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;aACvD,KAAK,EAAE,CAAC;IACb,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,aAAa,CAAC;IACxC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IACzB,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,IAAI,EAAE,EACR,CAAC,CAAC,MAAM,CAAC;QACP,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC;QAClB,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE;QACzB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;QACf,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC,CACH;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,OAAO,MAAM,GAAG,CAAC,EAAE;aAChB,KAAK,CAAC,OAAO,CAAC;aACd,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;aACjD,KAAK,EAAE,CAAC;IACb,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,YAAY,GAAG,gBAAgB,CAAC;IAC3C,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC9B;IACD,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC;IACzB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,qBAAqB,CAAC,CAAC;QAC9D,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;aAC1B,KAAK,CAAC,UAAU,CAAC;aACjB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;aACvD,KAAK,EAAE,CAAC;QACX,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,EAAE;gBAC3C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,GAAG;gBACd,SAAS;gBACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACtB,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,GAAG,CAAC;QACtB,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,EAAE;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,GAAG;YACd,SAAS;SACV,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,CAAC;IACzC,IAAI,EAAE;QACJ,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;QACf,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC9B;IACD,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC;IACtB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,mBAAmB,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;aAC1B,KAAK,CAAC,OAAO,CAAC;aACd,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;aACjD,KAAK,EAAE,CAAC;QACX,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,EAAE;gBACxC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,SAAS,EAAE,GAAG;gBACd,SAAS;aACV,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,GAAG,CAAC;QACtB,CAAC;QACD,OAAO,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE;YAClC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,GAAG;YACd,SAAS;SACV,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;IAClD,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE;IACjB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;aAC1B,KAAK,CAAC,UAAU,CAAC;aACjB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;aACvD,KAAK,EAAE,CAAC;QACX,IAAI,QAAQ;YAAE,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;QAC5D,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,eAAe,GAAG,gBAAgB,CAAC;IAC9C,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IACzB,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE;IACjB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,EAAE;aAC1B,KAAK,CAAC,OAAO,CAAC;aACd,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;aACjD,KAAK,EAAE,CAAC;QACX,IAAI,QAAQ;YAAE,MAAM,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAEH,gFAAgF;AAEhF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;IAC7B,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IAC3B,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,IAAI,EAAE,EACR,CAAC,CAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CACH;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,EAAE;aACrB,KAAK,CAAC,UAAU,CAAC;aACjB,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;aACvD,KAAK,EAAE,CAAC;QACX,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS;SACpC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,CAAC;IAC3B,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE;IACzB,OAAO,EAAE,CAAC,CAAC,KAAK,CACd,CAAC,CAAC,IAAI,EAAE,EACR,CAAC,CAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE;KACrB,CAAC,CACH;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,EAAE;aACrB,KAAK,CAAC,OAAO,CAAC;aACd,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;aACjD,KAAK,EAAE,CAAC;QACX,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS;SACpC,CAAC;IACJ,CAAC;CACF,CAAC,CAAC;AAEH,gFAAgF;AAEhF,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC;IAC3B,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC/B,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,+CAA+C;QAC/C,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;QAC9B,oBAAoB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC7C;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;QACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,iBAAiB;QACjB,iBAAiB;QACjB,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAChE,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAOM,CAAC;QACT,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,OAAO;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAED,qCAAqC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,oBAAoB,IAAI,WAAW,CAAC;QACtD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG;YACX,IAAI;YACJ,GAAG,EAAE,mCAAmC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,GACpE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAC5C,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;YACnD,MAAM,EAAE,MAAM;SACf,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;YAC3C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,kBAAkB,EAAE;aACnD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,+BACE,QAAQ,CAAC,MACX,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAC5B,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,oBAAoB;QACpB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE;YAC/C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IACvD,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC;IAC3B,IAAI,EAAE;QACJ,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;QACf,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE;QAC9B,yBAAyB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAClD;IACD,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;QACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,iBAAiB;QACjB,MAAM,MAAM,GAAG,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,EAAE;YAC5D,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAKM,CAAC;QACT,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAC5C,OAAO;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI;gBACf,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAED,yCAAyC;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,yBAAyB,IAAI,eAAe,CAAC;QAC/D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,oCAAoC,EAAE;YACjE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,kBAAkB,EAAE;aACnD;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,IAAI;gBACJ,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,MAAM,EAAE,KAAK;aACd,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CACb,mCACE,QAAQ,CAAC,MACX,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAC5B,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,oBAAoB;QACpB,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE;YAC7C,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,OAAO;YACP,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IACvD,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC;IAC/B,IAAI,EAAE;QACJ,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7B,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;KAC5B;IACD,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE;IACjB,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QAC3B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,mBAAmB,EAAE;gBACtD,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,eAAe,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
declare const _default: import("convex/server").SchemaDefinition<{
|
|
2
|
+
searches: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
3
|
+
recency?: string | undefined;
|
|
4
|
+
vertical?: string | undefined;
|
|
5
|
+
query: string;
|
|
6
|
+
results: string;
|
|
7
|
+
fetchedAt: number;
|
|
8
|
+
expiresAt: number;
|
|
9
|
+
}, {
|
|
10
|
+
query: import("convex/values").VString<string, "required">;
|
|
11
|
+
vertical: import("convex/values").VString<string | undefined, "optional">;
|
|
12
|
+
recency: import("convex/values").VString<string | undefined, "optional">;
|
|
13
|
+
results: import("convex/values").VString<string, "required">;
|
|
14
|
+
fetchedAt: import("convex/values").VFloat64<number, "required">;
|
|
15
|
+
expiresAt: import("convex/values").VFloat64<number, "required">;
|
|
16
|
+
}, "required", "query" | "recency" | "vertical" | "results" | "fetchedAt" | "expiresAt">, {
|
|
17
|
+
by_query: ["query", "_creationTime"];
|
|
18
|
+
by_expires: ["expiresAt", "_creationTime"];
|
|
19
|
+
}, {}, {}>;
|
|
20
|
+
pages: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
21
|
+
url: string;
|
|
22
|
+
fetchedAt: number;
|
|
23
|
+
expiresAt: number;
|
|
24
|
+
content: string;
|
|
25
|
+
}, {
|
|
26
|
+
url: import("convex/values").VString<string, "required">;
|
|
27
|
+
content: import("convex/values").VString<string, "required">;
|
|
28
|
+
fetchedAt: import("convex/values").VFloat64<number, "required">;
|
|
29
|
+
expiresAt: import("convex/values").VFloat64<number, "required">;
|
|
30
|
+
}, "required", "url" | "fetchedAt" | "expiresAt" | "content">, {
|
|
31
|
+
by_url: ["url", "_creationTime"];
|
|
32
|
+
by_expires: ["expiresAt", "_creationTime"];
|
|
33
|
+
}, {}, {}>;
|
|
34
|
+
processedRequests: import("convex/server").TableDefinition<import("convex/values").VObject<{
|
|
35
|
+
type: "search" | "scrape";
|
|
36
|
+
requestKey: string;
|
|
37
|
+
completedAt: number;
|
|
38
|
+
}, {
|
|
39
|
+
requestKey: import("convex/values").VString<string, "required">;
|
|
40
|
+
type: import("convex/values").VUnion<"search" | "scrape", [import("convex/values").VLiteral<"search", "required">, import("convex/values").VLiteral<"scrape", "required">], "required", never>;
|
|
41
|
+
completedAt: import("convex/values").VFloat64<number, "required">;
|
|
42
|
+
}, "required", "type" | "requestKey" | "completedAt">, {
|
|
43
|
+
by_key: ["requestKey", "_creationTime"];
|
|
44
|
+
}, {}, {}>;
|
|
45
|
+
}, true>;
|
|
46
|
+
export default _default;
|
|
47
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA,wBA0BG"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { defineSchema, defineTable } from "convex/server";
|
|
2
|
+
import { v } from "convex/values";
|
|
3
|
+
export default defineSchema({
|
|
4
|
+
searches: defineTable({
|
|
5
|
+
query: v.string(),
|
|
6
|
+
vertical: v.optional(v.string()), // "web" | "news" | "shopping" etc.
|
|
7
|
+
recency: v.optional(v.string()), // "d" | "w" | "m" etc.
|
|
8
|
+
results: v.string(), // JSON-stringified array of results
|
|
9
|
+
fetchedAt: v.number(), // Date.now()
|
|
10
|
+
expiresAt: v.number(), // fetchedAt + TTL
|
|
11
|
+
})
|
|
12
|
+
.index("by_query", ["query"])
|
|
13
|
+
.index("by_expires", ["expiresAt"]),
|
|
14
|
+
pages: defineTable({
|
|
15
|
+
url: v.string(),
|
|
16
|
+
content: v.string(), // markdown from Web Unlocker
|
|
17
|
+
fetchedAt: v.number(),
|
|
18
|
+
expiresAt: v.number(),
|
|
19
|
+
})
|
|
20
|
+
.index("by_url", ["url"])
|
|
21
|
+
.index("by_expires", ["expiresAt"]),
|
|
22
|
+
processedRequests: defineTable({
|
|
23
|
+
requestKey: v.string(), // hash of query+vertical+recency OR url
|
|
24
|
+
type: v.union(v.literal("search"), v.literal("scrape")),
|
|
25
|
+
completedAt: v.number(),
|
|
26
|
+
}).index("by_key", ["requestKey"]),
|
|
27
|
+
});
|
|
28
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/component/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAElC,eAAe,YAAY,CAAC;IAC1B,QAAQ,EAAE,WAAW,CAAC;QACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,mCAAmC;QACrE,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAG,uBAAuB;QACzD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAe,oCAAoC;QACtE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAa,aAAa;QAC/C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAa,kBAAkB;KACrD,CAAC;SACC,KAAK,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,CAAC;SAC5B,KAAK,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,CAAC;IAErC,KAAK,EAAE,WAAW,CAAC;QACjB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE;QACf,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAI,6BAA6B;QACpD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;KACtB,CAAC;SACC,KAAK,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;SACxB,KAAK,CAAC,YAAY,EAAE,CAAC,WAAW,CAAC,CAAC;IAErC,iBAAiB,EAAE,WAAW,CAAC;QAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,wCAAwC;QAChE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACvD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;KACxB,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,YAAY,CAAC,CAAC;CACnC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/react/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,cAAc,UAE1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/react/index.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;AAEb,6CAA6C;AAE7C,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,EAAE;IACjC,OAAO,EAAE,CAAC;AACZ,CAAC,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@sholajegede/bright-data-sync",
|
|
3
|
+
"description": "A bright data sync component for Convex.",
|
|
4
|
+
"repository": "github:sholajegede/bright-data-sync",
|
|
5
|
+
"homepage": "https://github.com/sholajegede/bright-data-sync#readme",
|
|
6
|
+
"bugs": {
|
|
7
|
+
"url": "https://github.com/sholajegede/bright-data-sync/issues"
|
|
8
|
+
},
|
|
9
|
+
"version": "0.1.1-alpha.0",
|
|
10
|
+
"license": "Apache-2.0",
|
|
11
|
+
"keywords": [
|
|
12
|
+
"convex",
|
|
13
|
+
"component"
|
|
14
|
+
],
|
|
15
|
+
"type": "module",
|
|
16
|
+
"scripts": {
|
|
17
|
+
"dev": "run-p -r 'dev:*'",
|
|
18
|
+
"dev:backend": "convex dev --typecheck-components",
|
|
19
|
+
"dev:frontend": "cd example && vite --clearScreen false",
|
|
20
|
+
"dev:build": "chokidar 'tsconfig*.json' 'src/**/*.ts' -i '**/*.test.ts' -c 'npm run build:codegen' --initial",
|
|
21
|
+
"predev": "path-exists .env.local dist || (npm run build && convex dev --once)",
|
|
22
|
+
"build": "tsc --project ./tsconfig.build.json",
|
|
23
|
+
"build:codegen": "npx convex codegen --component-dir ./src/component && npm run build",
|
|
24
|
+
"build:clean": "rm -rf dist *.tsbuildinfo && npm run build:codegen",
|
|
25
|
+
"typecheck": "tsc --noEmit && tsc -p example && tsc -p example/convex",
|
|
26
|
+
"lint": "eslint .",
|
|
27
|
+
"all": "run-p -r 'dev:*' 'test:watch'",
|
|
28
|
+
"test": "vitest run --typecheck",
|
|
29
|
+
"test:watch": "vitest --typecheck --clearScreen false",
|
|
30
|
+
"test:debug": "vitest --inspect-brk --no-file-parallelism",
|
|
31
|
+
"test:coverage": "vitest run --coverage --coverage.reporter=text",
|
|
32
|
+
"preversion": "npm ci && npm run build:clean && run-p test lint typecheck",
|
|
33
|
+
"prepublishOnly": "npm whoami || npm login",
|
|
34
|
+
"alpha": "npm version prerelease --preid alpha && npm publish --tag alpha && git push --follow-tags",
|
|
35
|
+
"release": "npm version patch && npm publish && git push --follow-tags",
|
|
36
|
+
"version": "vim -c 'normal o' -c 'normal o## '$npm_package_version CHANGELOG.md && prettier -w CHANGELOG.md && git add CHANGELOG.md"
|
|
37
|
+
},
|
|
38
|
+
"files": [
|
|
39
|
+
"dist",
|
|
40
|
+
"src"
|
|
41
|
+
],
|
|
42
|
+
"exports": {
|
|
43
|
+
"./package.json": "./package.json",
|
|
44
|
+
".": {
|
|
45
|
+
"types": "./dist/client/index.d.ts",
|
|
46
|
+
"default": "./dist/client/index.js"
|
|
47
|
+
},
|
|
48
|
+
"./react": {
|
|
49
|
+
"types": "./dist/react/index.d.ts",
|
|
50
|
+
"default": "./dist/react/index.js"
|
|
51
|
+
},
|
|
52
|
+
"./test": "./src/test.ts",
|
|
53
|
+
"./_generated/component.js": {
|
|
54
|
+
"types": "./dist/component/_generated/component.d.ts"
|
|
55
|
+
},
|
|
56
|
+
"./_generated/component": {
|
|
57
|
+
"types": "./dist/component/_generated/component.d.ts"
|
|
58
|
+
},
|
|
59
|
+
"./convex.config.js": {
|
|
60
|
+
"types": "./dist/component/convex.config.d.ts",
|
|
61
|
+
"default": "./dist/component/convex.config.js"
|
|
62
|
+
},
|
|
63
|
+
"./convex.config": {
|
|
64
|
+
"types": "./dist/component/convex.config.d.ts",
|
|
65
|
+
"default": "./dist/component/convex.config.js"
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"peerDependencies": {
|
|
69
|
+
"convex": "^1.32.0",
|
|
70
|
+
"react": "^18.3.1 || ^19.0.0"
|
|
71
|
+
},
|
|
72
|
+
"devDependencies": {
|
|
73
|
+
"@convex-dev/eslint-plugin": "^1.1.1",
|
|
74
|
+
"@edge-runtime/vm": "^5.0.0",
|
|
75
|
+
"@eslint/eslintrc": "^3.3.5",
|
|
76
|
+
"@eslint/js": "9.39.4",
|
|
77
|
+
"@types/node": "^24.12.0",
|
|
78
|
+
"@types/react": "^19.2.14",
|
|
79
|
+
"@types/react-dom": "^19.2.3",
|
|
80
|
+
"@vitejs/plugin-react": "^5.1.4",
|
|
81
|
+
"chokidar-cli": "3.0.0",
|
|
82
|
+
"convex": "1.32.0",
|
|
83
|
+
"convex-test": "0.0.41",
|
|
84
|
+
"eslint": "9.39.3",
|
|
85
|
+
"eslint-plugin-react": "^7.37.5",
|
|
86
|
+
"eslint-plugin-react-hooks": "^7.0.1",
|
|
87
|
+
"eslint-plugin-react-refresh": "^0.5.2",
|
|
88
|
+
"globals": "^17.4.0",
|
|
89
|
+
"npm-run-all2": "8.0.4",
|
|
90
|
+
"path-exists-cli": "2.0.0",
|
|
91
|
+
"pkg-pr-new": "^0.0.65",
|
|
92
|
+
"prettier": "3.8.1",
|
|
93
|
+
"react": "^19.2.4",
|
|
94
|
+
"react-dom": "^19.2.4",
|
|
95
|
+
"typescript": "5.9.3",
|
|
96
|
+
"typescript-eslint": "8.56.1",
|
|
97
|
+
"vite": "7.3.1",
|
|
98
|
+
"vitest": "4.0.18"
|
|
99
|
+
},
|
|
100
|
+
"types": "./dist/client/index.d.ts",
|
|
101
|
+
"module": "./dist/client/index.js"
|
|
102
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
// This is only here so convex-test can detect a _generated folder
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { describe, expect, test } from "vitest";
|
|
2
|
+
import { BrightDataSync } from "./index.js";
|
|
3
|
+
import { components } from "./setup.test.js";
|
|
4
|
+
|
|
5
|
+
describe("BrightDataSync client", () => {
|
|
6
|
+
test("instantiates with required options", () => {
|
|
7
|
+
const client = new BrightDataSync(components.brightDataSync, {
|
|
8
|
+
BRIGHTDATA_API_TOKEN: "test-token",
|
|
9
|
+
});
|
|
10
|
+
expect(client).toBeDefined();
|
|
11
|
+
expect(client.component).toBeDefined();
|
|
12
|
+
});
|
|
13
|
+
});
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { GenericActionCtx, GenericDataModel } from "convex/server";
|
|
2
|
+
import type { ComponentApi } from "../component/_generated/component.js";
|
|
3
|
+
|
|
4
|
+
export type BrightDataSyncOptions = {
|
|
5
|
+
/** Your Bright Data API token */
|
|
6
|
+
BRIGHTDATA_API_TOKEN: string;
|
|
7
|
+
/** Bright Data SERP zone name (default: "serp_api1") */
|
|
8
|
+
BRIGHTDATA_SEARCH_ZONE?: string;
|
|
9
|
+
/** Bright Data Web Unlocker zone name (default: "web_unlocker1") */
|
|
10
|
+
BRIGHTDATA_WEB_UNLOCKER_ZONE?: string;
|
|
11
|
+
/** Default TTL for search results in ms (default: 1 hour) */
|
|
12
|
+
DEFAULT_SEARCH_TTL_MS?: number;
|
|
13
|
+
/** Default TTL for scraped pages in ms (default: 6 hours) */
|
|
14
|
+
DEFAULT_PAGE_TTL_MS?: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export type SearchArgs = {
|
|
18
|
+
query: string;
|
|
19
|
+
vertical?: string;
|
|
20
|
+
recency?: string;
|
|
21
|
+
ttlMs?: number;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export type ScrapeArgs = {
|
|
25
|
+
url: string;
|
|
26
|
+
ttlMs?: number;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Client wrapper for the BrightDataSync Convex component.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* // convex/brightData.ts
|
|
35
|
+
* import { components } from "./_generated/api.js";
|
|
36
|
+
* import { BrightDataSync } from "@sholajegede/bright-data-sync";
|
|
37
|
+
*
|
|
38
|
+
* export const brightData = new BrightDataSync(components.brightDataSync, {
|
|
39
|
+
* BRIGHTDATA_API_TOKEN: process.env.BRIGHTDATA_API_TOKEN!,
|
|
40
|
+
* });
|
|
41
|
+
*
|
|
42
|
+
* // In an action:
|
|
43
|
+
* export const mySearch = action({
|
|
44
|
+
* args: { q: v.string() },
|
|
45
|
+
* handler: async (ctx, args) => {
|
|
46
|
+
* return await brightData.search(ctx, { query: args.q });
|
|
47
|
+
* },
|
|
48
|
+
* });
|
|
49
|
+
* ```
|
|
50
|
+
*/
|
|
51
|
+
export class BrightDataSync {
|
|
52
|
+
constructor(
|
|
53
|
+
public component: ComponentApi,
|
|
54
|
+
private options: BrightDataSyncOptions,
|
|
55
|
+
) {}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Search via Bright Data SERP API. Returns cached results if fresh.
|
|
59
|
+
*/
|
|
60
|
+
async search(ctx: ActionCtx, args: SearchArgs) {
|
|
61
|
+
return ctx.runAction(this.component.lib.search, {
|
|
62
|
+
query: args.query,
|
|
63
|
+
vertical: args.vertical,
|
|
64
|
+
recency: args.recency,
|
|
65
|
+
ttlMs: args.ttlMs ?? this.options.DEFAULT_SEARCH_TTL_MS,
|
|
66
|
+
brightdataApiToken: this.options.BRIGHTDATA_API_TOKEN,
|
|
67
|
+
brightdataSearchZone: this.options.BRIGHTDATA_SEARCH_ZONE,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Scrape a URL via Bright Data Web Unlocker. Returns cached content if fresh.
|
|
73
|
+
*/
|
|
74
|
+
async scrape(ctx: ActionCtx, args: ScrapeArgs) {
|
|
75
|
+
return ctx.runAction(this.component.lib.scrape, {
|
|
76
|
+
url: args.url,
|
|
77
|
+
ttlMs: args.ttlMs ?? this.options.DEFAULT_PAGE_TTL_MS,
|
|
78
|
+
brightdataApiToken: this.options.BRIGHTDATA_API_TOKEN,
|
|
79
|
+
brightdataWebUnlockerZone: this.options.BRIGHTDATA_WEB_UNLOCKER_ZONE,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Invalidate cached search results or page content.
|
|
85
|
+
*/
|
|
86
|
+
async invalidate(
|
|
87
|
+
ctx: ActionCtx,
|
|
88
|
+
args: { query?: string; url?: string },
|
|
89
|
+
) {
|
|
90
|
+
return ctx.runAction(this.component.lib.invalidate, args);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Re-export the component API type for convenience
|
|
95
|
+
export type { ComponentApi };
|
|
96
|
+
|
|
97
|
+
type ActionCtx = Pick<
|
|
98
|
+
GenericActionCtx<GenericDataModel>,
|
|
99
|
+
"runQuery" | "runMutation" | "runAction"
|
|
100
|
+
>;
|