@lobehub/lobehub 2.0.0-next.36 → 2.0.0-next.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/changelog/v1.json +18 -0
- package/locales/ar/modelProvider.json +13 -1
- package/locales/bg-BG/modelProvider.json +13 -1
- package/locales/de-DE/modelProvider.json +13 -1
- package/locales/en-US/modelProvider.json +13 -1
- package/locales/es-ES/modelProvider.json +13 -1
- package/locales/fa-IR/modelProvider.json +13 -1
- package/locales/fr-FR/modelProvider.json +13 -1
- package/locales/it-IT/modelProvider.json +13 -1
- package/locales/ja-JP/modelProvider.json +13 -1
- package/locales/ko-KR/modelProvider.json +13 -1
- package/locales/nl-NL/modelProvider.json +13 -1
- package/locales/pl-PL/modelProvider.json +13 -1
- package/locales/pt-BR/modelProvider.json +13 -1
- package/locales/ru-RU/modelProvider.json +13 -1
- package/locales/tr-TR/modelProvider.json +13 -1
- package/locales/vi-VN/modelProvider.json +13 -1
- package/locales/zh-CN/modelProvider.json +13 -1
- package/locales/zh-TW/modelProvider.json +13 -1
- package/package.json +1 -1
- package/packages/database/src/models/__tests__/apiKey.test.ts +444 -0
- package/packages/web-crawler/src/crawImpl/browserless.ts +1 -1
- package/packages/web-crawler/src/crawImpl/naive.ts +9 -9
- package/packages/web-crawler/src/crawler.ts +5 -5
- package/packages/web-crawler/src/urlRules.ts +13 -13
- package/packages/web-crawler/src/utils/appUrlRules.ts +5 -5
- package/src/features/Conversation/components/Extras/Usage/UsageDetail/index.tsx +3 -0
- package/src/libs/trpc/client/lambda.ts +4 -3
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
// @vitest-environment node
|
|
2
|
+
import { eq } from 'drizzle-orm';
|
|
3
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
4
|
+
|
|
5
|
+
import { apiKeys, users } from '../../schemas';
|
|
6
|
+
import { LobeChatDatabase } from '../../type';
|
|
7
|
+
import { ApiKeyModel } from '../apiKey';
|
|
8
|
+
import { getTestDB } from './_util';
|
|
9
|
+
|
|
10
|
+
const serverDB: LobeChatDatabase = await getTestDB();
|
|
11
|
+
|
|
12
|
+
const userId = 'api-key-model-test-user-id';
|
|
13
|
+
const apiKeyModel = new ApiKeyModel(serverDB, userId);
|
|
14
|
+
|
|
15
|
+
beforeEach(async () => {
|
|
16
|
+
await serverDB.delete(users);
|
|
17
|
+
await serverDB.insert(users).values([{ id: userId }, { id: 'user2' }]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
afterEach(async () => {
|
|
21
|
+
await serverDB.delete(users).where(eq(users.id, userId));
|
|
22
|
+
await serverDB.delete(apiKeys).where(eq(apiKeys.userId, userId));
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
describe('ApiKeyModel', () => {
|
|
26
|
+
describe('create', () => {
|
|
27
|
+
it('should create a new API key without encryption', async () => {
|
|
28
|
+
const params = {
|
|
29
|
+
enabled: true,
|
|
30
|
+
name: 'Test API Key',
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const result = await apiKeyModel.create(params);
|
|
34
|
+
|
|
35
|
+
expect(result.id).toBeDefined();
|
|
36
|
+
expect(result.name).toBe(params.name);
|
|
37
|
+
expect(result.enabled).toBe(params.enabled);
|
|
38
|
+
expect(result.key).toBeDefined();
|
|
39
|
+
expect(result.key).toMatch(/^lb-[\da-z]{16}$/);
|
|
40
|
+
expect(result.userId).toBe(userId);
|
|
41
|
+
|
|
42
|
+
const apiKey = await serverDB.query.apiKeys.findFirst({
|
|
43
|
+
where: eq(apiKeys.id, result.id),
|
|
44
|
+
});
|
|
45
|
+
expect(apiKey).toMatchObject({ ...params, userId });
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('should create a new API key with encryption', async () => {
|
|
49
|
+
const mockEncryptor = vi.fn().mockResolvedValue('encrypted-key-value');
|
|
50
|
+
const params = {
|
|
51
|
+
enabled: true,
|
|
52
|
+
name: 'Encrypted API Key',
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
const result = await apiKeyModel.create(params, mockEncryptor);
|
|
56
|
+
|
|
57
|
+
expect(result.id).toBeDefined();
|
|
58
|
+
expect(result.name).toBe(params.name);
|
|
59
|
+
expect(result.key).toBe('encrypted-key-value');
|
|
60
|
+
expect(mockEncryptor).toHaveBeenCalledWith(expect.stringMatching(/^lb-[\da-z]{16}$/));
|
|
61
|
+
|
|
62
|
+
const apiKey = await serverDB.query.apiKeys.findFirst({
|
|
63
|
+
where: eq(apiKeys.id, result.id),
|
|
64
|
+
});
|
|
65
|
+
expect(apiKey?.key).toBe('encrypted-key-value');
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
it('should create API key with expiration date', async () => {
|
|
69
|
+
const expiresAt = new Date('2025-12-31');
|
|
70
|
+
const params = {
|
|
71
|
+
enabled: true,
|
|
72
|
+
expiresAt,
|
|
73
|
+
name: 'Expiring Key',
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
const result = await apiKeyModel.create(params);
|
|
77
|
+
|
|
78
|
+
expect(result.expiresAt).toEqual(expiresAt);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
describe('delete', () => {
|
|
83
|
+
it('should delete an API key by id', async () => {
|
|
84
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
85
|
+
|
|
86
|
+
await apiKeyModel.delete(id);
|
|
87
|
+
|
|
88
|
+
const apiKey = await serverDB.query.apiKeys.findFirst({
|
|
89
|
+
where: eq(apiKeys.id, id),
|
|
90
|
+
});
|
|
91
|
+
expect(apiKey).toBeUndefined();
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('should only delete API keys for the current user', async () => {
|
|
95
|
+
const { id: key1 } = await apiKeyModel.create({ name: 'User 1 Key', enabled: true });
|
|
96
|
+
|
|
97
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
98
|
+
const { id: key2 } = await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
99
|
+
|
|
100
|
+
await apiKeyModel.delete(key2);
|
|
101
|
+
|
|
102
|
+
const key2Still = await serverDB.query.apiKeys.findFirst({
|
|
103
|
+
where: eq(apiKeys.id, key2),
|
|
104
|
+
});
|
|
105
|
+
expect(key2Still).toBeDefined();
|
|
106
|
+
|
|
107
|
+
await apiKeyModel.delete(key1);
|
|
108
|
+
|
|
109
|
+
const key1Deleted = await serverDB.query.apiKeys.findFirst({
|
|
110
|
+
where: eq(apiKeys.id, key1),
|
|
111
|
+
});
|
|
112
|
+
expect(key1Deleted).toBeUndefined();
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe('deleteAll', () => {
|
|
117
|
+
it('should delete all API keys for the user', async () => {
|
|
118
|
+
await apiKeyModel.create({ name: 'Test Key 1', enabled: true });
|
|
119
|
+
await apiKeyModel.create({ name: 'Test Key 2', enabled: true });
|
|
120
|
+
|
|
121
|
+
await apiKeyModel.deleteAll();
|
|
122
|
+
|
|
123
|
+
const userKeys = await serverDB.query.apiKeys.findMany({
|
|
124
|
+
where: eq(apiKeys.userId, userId),
|
|
125
|
+
});
|
|
126
|
+
expect(userKeys).toHaveLength(0);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('should only delete API keys for the user, not others', async () => {
|
|
130
|
+
await apiKeyModel.create({ name: 'Test Key 1', enabled: true });
|
|
131
|
+
await apiKeyModel.create({ name: 'Test Key 2', enabled: true });
|
|
132
|
+
|
|
133
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
134
|
+
await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
135
|
+
|
|
136
|
+
await apiKeyModel.deleteAll();
|
|
137
|
+
|
|
138
|
+
const userKeys = await serverDB.query.apiKeys.findMany({
|
|
139
|
+
where: eq(apiKeys.userId, userId),
|
|
140
|
+
});
|
|
141
|
+
const total = await serverDB.query.apiKeys.findMany();
|
|
142
|
+
expect(userKeys).toHaveLength(0);
|
|
143
|
+
expect(total).toHaveLength(1);
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
describe('query', () => {
|
|
148
|
+
it('should query API keys for the user without decryption', async () => {
|
|
149
|
+
await apiKeyModel.create({ name: 'Key 1', enabled: true });
|
|
150
|
+
await apiKeyModel.create({ name: 'Key 2', enabled: true });
|
|
151
|
+
|
|
152
|
+
const keys = await apiKeyModel.query();
|
|
153
|
+
expect(keys).toHaveLength(2);
|
|
154
|
+
expect(keys[0].key).toMatch(/^lb-[\da-z]{16}$/);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it('should query API keys ordered by updatedAt desc', async () => {
|
|
158
|
+
const key1 = await apiKeyModel.create({ name: 'Key 1', enabled: true });
|
|
159
|
+
// Wait a bit to ensure different timestamps
|
|
160
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
161
|
+
const key2 = await apiKeyModel.create({ name: 'Key 2', enabled: true });
|
|
162
|
+
|
|
163
|
+
const keys = await apiKeyModel.query();
|
|
164
|
+
expect(keys).toHaveLength(2);
|
|
165
|
+
expect(keys[0].id).toBe(key2.id);
|
|
166
|
+
expect(keys[1].id).toBe(key1.id);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it('should query API keys with decryption', async () => {
|
|
170
|
+
const mockEncryptor = vi.fn().mockResolvedValue('encrypted-key');
|
|
171
|
+
const mockDecryptor = vi.fn().mockResolvedValue({ plaintext: 'decrypted-key-value' });
|
|
172
|
+
|
|
173
|
+
await apiKeyModel.create({ name: 'Encrypted Key', enabled: true }, mockEncryptor);
|
|
174
|
+
|
|
175
|
+
const keys = await apiKeyModel.query(mockDecryptor);
|
|
176
|
+
|
|
177
|
+
expect(keys).toHaveLength(1);
|
|
178
|
+
expect(keys[0].key).toBe('decrypted-key-value');
|
|
179
|
+
expect(mockDecryptor).toHaveBeenCalledWith('encrypted-key');
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it('should only query API keys for the current user', async () => {
|
|
183
|
+
await apiKeyModel.create({ name: 'User 1 Key', enabled: true });
|
|
184
|
+
|
|
185
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
186
|
+
await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
187
|
+
|
|
188
|
+
const keys = await apiKeyModel.query();
|
|
189
|
+
expect(keys).toHaveLength(1);
|
|
190
|
+
expect(keys[0].name).toBe('User 1 Key');
|
|
191
|
+
});
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
describe('findByKey', () => {
|
|
195
|
+
it('should find API key by key value without encryption', async () => {
|
|
196
|
+
// Use a valid hex format key since validateApiKeyFormat checks for hex pattern
|
|
197
|
+
const validKey = 'lb-abcdef0123456789';
|
|
198
|
+
await serverDB.insert(apiKeys).values({
|
|
199
|
+
enabled: true,
|
|
200
|
+
key: validKey,
|
|
201
|
+
name: 'Test Key',
|
|
202
|
+
userId,
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
const found = await apiKeyModel.findByKey(validKey);
|
|
206
|
+
|
|
207
|
+
expect(found).toBeDefined();
|
|
208
|
+
expect(found?.key).toBe(validKey);
|
|
209
|
+
expect(found?.name).toBe('Test Key');
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
it('should find API key by key value with encryption', async () => {
|
|
213
|
+
const mockEncryptor = vi.fn().mockResolvedValue('encrypted-key-value');
|
|
214
|
+
const created = await apiKeyModel.create({ name: 'Test Key', enabled: true }, mockEncryptor);
|
|
215
|
+
|
|
216
|
+
const testKey = 'lb-0123456789abcdef';
|
|
217
|
+
mockEncryptor.mockResolvedValue('encrypted-key-value');
|
|
218
|
+
const found = await apiKeyModel.findByKey(testKey, mockEncryptor);
|
|
219
|
+
|
|
220
|
+
expect(mockEncryptor).toHaveBeenCalledWith(testKey);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it('should return null for invalid key format', async () => {
|
|
224
|
+
const found = await apiKeyModel.findByKey('invalid-key-format');
|
|
225
|
+
|
|
226
|
+
expect(found).toBeNull();
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
it('should return undefined for non-existent key', async () => {
|
|
230
|
+
const found = await apiKeyModel.findByKey('lb-0123456789abcdef');
|
|
231
|
+
|
|
232
|
+
expect(found).toBeUndefined();
|
|
233
|
+
});
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
describe('validateKey', () => {
|
|
237
|
+
it('should validate enabled and non-expired key with valid hex format', async () => {
|
|
238
|
+
const futureDate = new Date();
|
|
239
|
+
futureDate.setFullYear(futureDate.getFullYear() + 1);
|
|
240
|
+
|
|
241
|
+
// Use a valid hex format key
|
|
242
|
+
const validKey = 'lb-0123456789abcdef';
|
|
243
|
+
await serverDB.insert(apiKeys).values({
|
|
244
|
+
enabled: true,
|
|
245
|
+
expiresAt: futureDate,
|
|
246
|
+
key: validKey,
|
|
247
|
+
name: 'Valid Key',
|
|
248
|
+
userId,
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
const isValid = await apiKeyModel.validateKey(validKey);
|
|
252
|
+
|
|
253
|
+
expect(isValid).toBe(true);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
it('should validate enabled key without expiration with valid hex format', async () => {
|
|
257
|
+
// Use a valid hex format key
|
|
258
|
+
const validKey = 'lb-fedcba9876543210';
|
|
259
|
+
await serverDB.insert(apiKeys).values({
|
|
260
|
+
enabled: true,
|
|
261
|
+
key: validKey,
|
|
262
|
+
name: 'Valid Key',
|
|
263
|
+
userId,
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
const isValid = await apiKeyModel.validateKey(validKey);
|
|
267
|
+
|
|
268
|
+
expect(isValid).toBe(true);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it('should reject non-existent key', async () => {
|
|
272
|
+
const isValid = await apiKeyModel.validateKey('lb-0123456789abcdef');
|
|
273
|
+
|
|
274
|
+
expect(isValid).toBe(false);
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
it('should reject disabled key', async () => {
|
|
278
|
+
const validKey = 'lb-1111111111111111';
|
|
279
|
+
await serverDB.insert(apiKeys).values({
|
|
280
|
+
enabled: false,
|
|
281
|
+
key: validKey,
|
|
282
|
+
name: 'Disabled Key',
|
|
283
|
+
userId,
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
const isValid = await apiKeyModel.validateKey(validKey);
|
|
287
|
+
|
|
288
|
+
expect(isValid).toBe(false);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
it('should reject expired key', async () => {
|
|
292
|
+
const pastDate = new Date();
|
|
293
|
+
pastDate.setFullYear(pastDate.getFullYear() - 1);
|
|
294
|
+
|
|
295
|
+
const validKey = 'lb-2222222222222222';
|
|
296
|
+
await serverDB.insert(apiKeys).values({
|
|
297
|
+
enabled: true,
|
|
298
|
+
expiresAt: pastDate,
|
|
299
|
+
key: validKey,
|
|
300
|
+
name: 'Expired Key',
|
|
301
|
+
userId,
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
const isValid = await apiKeyModel.validateKey(validKey);
|
|
305
|
+
|
|
306
|
+
expect(isValid).toBe(false);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
it('should reject invalid key format', async () => {
|
|
310
|
+
const isValid = await apiKeyModel.validateKey('invalid-format');
|
|
311
|
+
|
|
312
|
+
expect(isValid).toBe(false);
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
describe('update', () => {
|
|
317
|
+
it('should update API key properties', async () => {
|
|
318
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
319
|
+
|
|
320
|
+
await apiKeyModel.update(id, { name: 'Updated Key', enabled: false });
|
|
321
|
+
|
|
322
|
+
const updated = await serverDB.query.apiKeys.findFirst({
|
|
323
|
+
where: eq(apiKeys.id, id),
|
|
324
|
+
});
|
|
325
|
+
expect(updated).toMatchObject({
|
|
326
|
+
enabled: false,
|
|
327
|
+
id,
|
|
328
|
+
name: 'Updated Key',
|
|
329
|
+
userId,
|
|
330
|
+
});
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
it('should update expiration date', async () => {
|
|
334
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
335
|
+
|
|
336
|
+
const newExpiresAt = new Date('2026-12-31');
|
|
337
|
+
await apiKeyModel.update(id, { expiresAt: newExpiresAt });
|
|
338
|
+
|
|
339
|
+
const updated = await serverDB.query.apiKeys.findFirst({
|
|
340
|
+
where: eq(apiKeys.id, id),
|
|
341
|
+
});
|
|
342
|
+
expect(updated?.expiresAt).toEqual(newExpiresAt);
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
it('should only update API keys for the current user', async () => {
|
|
346
|
+
const { id: key1 } = await apiKeyModel.create({ name: 'User 1 Key', enabled: true });
|
|
347
|
+
|
|
348
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
349
|
+
const { id: key2 } = await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
350
|
+
|
|
351
|
+
await apiKeyModel.update(key2, { name: 'Attempted Update' });
|
|
352
|
+
|
|
353
|
+
const key2Still = await serverDB.query.apiKeys.findFirst({
|
|
354
|
+
where: eq(apiKeys.id, key2),
|
|
355
|
+
});
|
|
356
|
+
expect(key2Still?.name).toBe('User 2 Key');
|
|
357
|
+
});
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
describe('findById', () => {
|
|
361
|
+
it('should find API key by id', async () => {
|
|
362
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
363
|
+
|
|
364
|
+
const found = await apiKeyModel.findById(id);
|
|
365
|
+
|
|
366
|
+
expect(found).toMatchObject({
|
|
367
|
+
enabled: true,
|
|
368
|
+
id,
|
|
369
|
+
name: 'Test Key',
|
|
370
|
+
userId,
|
|
371
|
+
});
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it('should return undefined for non-existent id', async () => {
|
|
375
|
+
const found = await apiKeyModel.findById(999_999);
|
|
376
|
+
|
|
377
|
+
expect(found).toBeUndefined();
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
it('should only find API keys for the current user', async () => {
|
|
381
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
382
|
+
const { id: key2 } = await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
383
|
+
|
|
384
|
+
const found = await apiKeyModel.findById(key2);
|
|
385
|
+
|
|
386
|
+
expect(found).toBeUndefined();
|
|
387
|
+
});
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
describe('updateLastUsed', () => {
|
|
391
|
+
it('should update lastUsedAt timestamp', async () => {
|
|
392
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
393
|
+
|
|
394
|
+
const beforeUpdate = await serverDB.query.apiKeys.findFirst({
|
|
395
|
+
where: eq(apiKeys.id, id),
|
|
396
|
+
});
|
|
397
|
+
expect(beforeUpdate?.lastUsedAt).toBeNull();
|
|
398
|
+
|
|
399
|
+
await apiKeyModel.updateLastUsed(id);
|
|
400
|
+
|
|
401
|
+
const afterUpdate = await serverDB.query.apiKeys.findFirst({
|
|
402
|
+
where: eq(apiKeys.id, id),
|
|
403
|
+
});
|
|
404
|
+
expect(afterUpdate?.lastUsedAt).toBeInstanceOf(Date);
|
|
405
|
+
});
|
|
406
|
+
|
|
407
|
+
it('should only update API keys for the current user', async () => {
|
|
408
|
+
const anotherApiKeyModel = new ApiKeyModel(serverDB, 'user2');
|
|
409
|
+
const { id: key2 } = await anotherApiKeyModel.create({ name: 'User 2 Key', enabled: true });
|
|
410
|
+
|
|
411
|
+
await apiKeyModel.updateLastUsed(key2);
|
|
412
|
+
|
|
413
|
+
const key2Still = await serverDB.query.apiKeys.findFirst({
|
|
414
|
+
where: eq(apiKeys.id, key2),
|
|
415
|
+
});
|
|
416
|
+
expect(key2Still?.lastUsedAt).toBeNull();
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
it('should update existing lastUsedAt to a new timestamp', async () => {
|
|
420
|
+
const { id } = await apiKeyModel.create({ name: 'Test Key', enabled: true });
|
|
421
|
+
|
|
422
|
+
await apiKeyModel.updateLastUsed(id);
|
|
423
|
+
|
|
424
|
+
const firstUpdate = await serverDB.query.apiKeys.findFirst({
|
|
425
|
+
where: eq(apiKeys.id, id),
|
|
426
|
+
});
|
|
427
|
+
const firstTimestamp = firstUpdate?.lastUsedAt;
|
|
428
|
+
|
|
429
|
+
// Wait to ensure different timestamp
|
|
430
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
431
|
+
|
|
432
|
+
await apiKeyModel.updateLastUsed(id);
|
|
433
|
+
|
|
434
|
+
const secondUpdate = await serverDB.query.apiKeys.findFirst({
|
|
435
|
+
where: eq(apiKeys.id, id),
|
|
436
|
+
});
|
|
437
|
+
const secondTimestamp = secondUpdate?.lastUsedAt;
|
|
438
|
+
|
|
439
|
+
expect(secondTimestamp).toBeDefined();
|
|
440
|
+
expect(firstTimestamp).toBeDefined();
|
|
441
|
+
expect(secondTimestamp!.getTime()).toBeGreaterThan(firstTimestamp!.getTime());
|
|
442
|
+
});
|
|
443
|
+
});
|
|
444
|
+
});
|
|
@@ -56,7 +56,7 @@ export const browserless: CrawlImpl = async (url, { filterOptions }) => {
|
|
|
56
56
|
if (
|
|
57
57
|
!!result.content &&
|
|
58
58
|
result.title &&
|
|
59
|
-
// Just a moment...
|
|
59
|
+
// "Just a moment..." indicates being blocked by CloudFlare
|
|
60
60
|
result.title.trim() !== 'Just a moment...'
|
|
61
61
|
) {
|
|
62
62
|
return {
|
|
@@ -6,25 +6,25 @@ import { htmlToMarkdown } from '../utils/htmlToMarkdown';
|
|
|
6
6
|
import { DEFAULT_TIMEOUT, withTimeout } from '../utils/withTimeout';
|
|
7
7
|
|
|
8
8
|
const mixinHeaders = {
|
|
9
|
-
//
|
|
9
|
+
// Accepted content types
|
|
10
10
|
'Accept':
|
|
11
11
|
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
|
12
|
-
//
|
|
12
|
+
// Accepted encoding methods
|
|
13
13
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
14
|
-
//
|
|
14
|
+
// Accepted languages
|
|
15
15
|
'Accept-Language': 'en-US,en;q=0.9,zh;q=0.8',
|
|
16
|
-
//
|
|
16
|
+
// Cache control
|
|
17
17
|
'Cache-Control': 'max-age=0',
|
|
18
|
-
//
|
|
18
|
+
// Connection type
|
|
19
19
|
'Connection': 'keep-alive',
|
|
20
|
-
//
|
|
20
|
+
// Indicates which site the request is from
|
|
21
21
|
'Referer': 'https://www.google.com/',
|
|
22
|
-
//
|
|
22
|
+
// Upgrade insecure requests
|
|
23
23
|
'Upgrade-Insecure-Requests': '1',
|
|
24
|
-
//
|
|
24
|
+
// Simulate real browser User-Agent
|
|
25
25
|
'User-Agent':
|
|
26
26
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
27
|
-
//
|
|
27
|
+
// Prevent cross-site request forgery
|
|
28
28
|
'sec-ch-ua': '"Google Chrome";v="121", "Not A(Brand";v="99", "Chromium";v="121"',
|
|
29
29
|
'sec-ch-ua-mobile': '?0',
|
|
30
30
|
'sec-ch-ua-platform': '"Windows"',
|
|
@@ -19,8 +19,8 @@ export class Crawler {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
23
|
-
* @param options
|
|
22
|
+
* Crawl webpage content
|
|
23
|
+
* @param options Crawl options
|
|
24
24
|
*/
|
|
25
25
|
async crawl({
|
|
26
26
|
url,
|
|
@@ -31,14 +31,14 @@ export class Crawler {
|
|
|
31
31
|
impls?: CrawlImplType[];
|
|
32
32
|
url: string;
|
|
33
33
|
}): Promise<CrawlUniformResult> {
|
|
34
|
-
//
|
|
34
|
+
// Apply URL rules
|
|
35
35
|
const {
|
|
36
36
|
transformedUrl,
|
|
37
37
|
filterOptions: ruleFilterOptions,
|
|
38
38
|
impls: ruleImpls,
|
|
39
39
|
} = applyUrlRules(url, crawUrlRules);
|
|
40
40
|
|
|
41
|
-
//
|
|
41
|
+
// Merge user-provided filter options and rule filter options, user options take priority
|
|
42
42
|
const mergedFilterOptions = {
|
|
43
43
|
...ruleFilterOptions,
|
|
44
44
|
...userFilterOptions,
|
|
@@ -53,7 +53,7 @@ export class Crawler {
|
|
|
53
53
|
? (userImpls.filter((impl) => Object.keys(crawlImpls).includes(impl)) as CrawlImplType[])
|
|
54
54
|
: systemImpls;
|
|
55
55
|
|
|
56
|
-
//
|
|
56
|
+
// Try each implementation in the built-in order
|
|
57
57
|
for (const impl of finalImpls) {
|
|
58
58
|
try {
|
|
59
59
|
const res = await crawlImpls[impl](transformedUrl, { filterOptions: mergedFilterOptions });
|
|
@@ -1,32 +1,32 @@
|
|
|
1
1
|
import { CrawlUrlRule } from './type';
|
|
2
2
|
|
|
3
3
|
export const crawUrlRules: CrawlUrlRule[] = [
|
|
4
|
-
//
|
|
4
|
+
// Sogou WeChat links, use search1api
|
|
5
5
|
{
|
|
6
6
|
impls: ['search1api'],
|
|
7
7
|
urlPattern: 'https://weixin.sogou.com/link(.*)',
|
|
8
8
|
},
|
|
9
|
-
//
|
|
9
|
+
// Sogou links, use search1api
|
|
10
10
|
{
|
|
11
11
|
impls: ['search1api'],
|
|
12
12
|
urlPattern: 'https://sogou.com/link(.*)',
|
|
13
13
|
},
|
|
14
|
-
// YouTube
|
|
14
|
+
// YouTube links, use search1api, formatted as markdown, can return subtitle content
|
|
15
15
|
{
|
|
16
16
|
impls: ['search1api'],
|
|
17
17
|
urlPattern: 'https://www.youtube.com/watch(.*)',
|
|
18
18
|
},
|
|
19
|
-
// Reddit
|
|
19
|
+
// Reddit links, use search1api, formatted as markdown, includes title, author, interaction count, specific comment content, etc.
|
|
20
20
|
{
|
|
21
21
|
impls: ['search1api'],
|
|
22
22
|
urlPattern: 'https://www.reddit.com/r/(.*)/comments/(.*)',
|
|
23
23
|
},
|
|
24
|
-
//
|
|
24
|
+
// WeChat official accounts have crawler protection, use search1api first, jina as fallback (currently jina crawling may be blocked)
|
|
25
25
|
{
|
|
26
26
|
impls: ['search1api', 'jina'],
|
|
27
27
|
urlPattern: 'https://mp.weixin.qq.com(.*)',
|
|
28
28
|
},
|
|
29
|
-
//
|
|
29
|
+
// GitHub source code parsing
|
|
30
30
|
{
|
|
31
31
|
filterOptions: {
|
|
32
32
|
enableReadability: false,
|
|
@@ -43,7 +43,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
|
|
|
43
43
|
// GitHub discussion
|
|
44
44
|
urlPattern: 'https://github.com/(.*)/discussions/(.*)',
|
|
45
45
|
},
|
|
46
|
-
//
|
|
46
|
+
// All PDFs use jina
|
|
47
47
|
{
|
|
48
48
|
impls: ['jina'],
|
|
49
49
|
urlPattern: 'https://(.*).pdf',
|
|
@@ -53,7 +53,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
|
|
|
53
53
|
impls: ['jina'],
|
|
54
54
|
urlPattern: 'https://arxiv.org/pdf/(.*)',
|
|
55
55
|
},
|
|
56
|
-
//
|
|
56
|
+
// Zhihu has crawler protection, use jina
|
|
57
57
|
{
|
|
58
58
|
impls: ['jina'],
|
|
59
59
|
urlPattern: 'https://zhuanlan.zhihu.com(.*)',
|
|
@@ -63,7 +63,7 @@ export const crawUrlRules: CrawlUrlRule[] = [
|
|
|
63
63
|
urlPattern: 'https://zhihu.com(.*)',
|
|
64
64
|
},
|
|
65
65
|
{
|
|
66
|
-
// Medium
|
|
66
|
+
// Convert Medium articles to Scribe.rip
|
|
67
67
|
urlPattern: 'https://medium.com/(.*)',
|
|
68
68
|
urlTransform: 'https://scribe.rip/$1',
|
|
69
69
|
},
|
|
@@ -74,10 +74,10 @@ export const crawUrlRules: CrawlUrlRule[] = [
|
|
|
74
74
|
impls: ['jina', 'browserless'],
|
|
75
75
|
urlPattern: 'https://(twitter.com|x.com)/(.*)',
|
|
76
76
|
},
|
|
77
|
-
//
|
|
77
|
+
// Sports data website rules
|
|
78
78
|
{
|
|
79
79
|
filterOptions: {
|
|
80
|
-
//
|
|
80
|
+
// Disable Readability for sports data tables and convert to plain text
|
|
81
81
|
enableReadability: false,
|
|
82
82
|
pureText: true,
|
|
83
83
|
},
|
|
@@ -94,13 +94,13 @@ export const crawUrlRules: CrawlUrlRule[] = [
|
|
|
94
94
|
impls: ['jina'],
|
|
95
95
|
urlPattern: 'https://cvpr.thecvf.com(.*)',
|
|
96
96
|
},
|
|
97
|
-
//
|
|
97
|
+
// Feishu use jina
|
|
98
98
|
// https://github.com/lobehub/lobe-chat/issues/6879
|
|
99
99
|
{
|
|
100
100
|
impls: ['jina'],
|
|
101
101
|
urlPattern: 'https://(.*).feishu.cn/(.*)',
|
|
102
102
|
},
|
|
103
|
-
//
|
|
103
|
+
// Xiaohongshu has crawler protection, use Search1API or Jina (fallback)
|
|
104
104
|
{
|
|
105
105
|
impls: ['search1api', 'jina'],
|
|
106
106
|
urlPattern: 'https://(.*).xiaohongshu.com/(.*)',
|
|
@@ -9,14 +9,14 @@ export const applyUrlRules = (
|
|
|
9
9
|
transformedUrl: string;
|
|
10
10
|
} => {
|
|
11
11
|
for (const rule of urlRules) {
|
|
12
|
-
//
|
|
12
|
+
// Convert to regular expression
|
|
13
13
|
const regex = new RegExp(rule.urlPattern);
|
|
14
14
|
const match = url.match(regex);
|
|
15
15
|
|
|
16
16
|
if (match) {
|
|
17
17
|
if (rule.urlTransform) {
|
|
18
|
-
//
|
|
19
|
-
//
|
|
18
|
+
// If there is a transformation rule, perform URL transformation
|
|
19
|
+
// Replace placeholders like $1, $2 with capture group content
|
|
20
20
|
const transformedUrl = rule.urlTransform.replaceAll(
|
|
21
21
|
/\$(\d+)/g,
|
|
22
22
|
(_, index) => match[parseInt(index)] || '',
|
|
@@ -28,7 +28,7 @@ export const applyUrlRules = (
|
|
|
28
28
|
transformedUrl,
|
|
29
29
|
};
|
|
30
30
|
} else {
|
|
31
|
-
//
|
|
31
|
+
// No transformation rule but pattern matched, only return filter options
|
|
32
32
|
return {
|
|
33
33
|
filterOptions: rule.filterOptions,
|
|
34
34
|
impls: rule.impls,
|
|
@@ -38,6 +38,6 @@ export const applyUrlRules = (
|
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
-
//
|
|
41
|
+
// No rule matched, return original URL
|
|
42
42
|
return { transformedUrl: url };
|
|
43
43
|
};
|
|
@@ -217,6 +217,9 @@ const TokenDetail = memo<TokenDetailProps>(({ meta, model, provider }) => {
|
|
|
217
217
|
<Icon icon={isShowCredit ? BadgeCent : CoinsIcon} />
|
|
218
218
|
<AnimatedNumber
|
|
219
219
|
formatter={(value) => (formatShortenNumber(value) as string).toLowerCase?.()}
|
|
220
|
+
// Force remount when switching between token/credit to prevent unwanted animation
|
|
221
|
+
// See: https://github.com/lobehub/lobe-chat/pull/10098
|
|
222
|
+
key={isShowCredit ? 'credit' : 'token'}
|
|
220
223
|
value={totalCount}
|
|
221
224
|
/>
|
|
222
225
|
</Center>
|
|
@@ -84,7 +84,7 @@ const customHttpBatchLink = httpBatchLink({
|
|
|
84
84
|
// dynamic import to avoid circular dependency
|
|
85
85
|
const { createHeaderWithAuth } = await import('@/services/_auth');
|
|
86
86
|
|
|
87
|
-
let provider: ModelProvider
|
|
87
|
+
let provider: ModelProvider | undefined;
|
|
88
88
|
// for image page, we need to get the provider from the store
|
|
89
89
|
log('Getting provider from store for image page: %s', location.pathname);
|
|
90
90
|
if (location.pathname === '/image') {
|
|
@@ -96,8 +96,9 @@ const customHttpBatchLink = httpBatchLink({
|
|
|
96
96
|
log('Getting provider from store for image page: %s', provider);
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
-
//
|
|
100
|
-
|
|
99
|
+
// Only include provider in JWT for image operations
|
|
100
|
+
// For other operations (like knowledge base embedding), let server use its own config
|
|
101
|
+
const headers = await createHeaderWithAuth(provider ? { provider } : undefined);
|
|
101
102
|
log('Headers: %O', headers);
|
|
102
103
|
return headers;
|
|
103
104
|
},
|