@s-hirano-ist/s-scripts 1.5.2 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/find-duplicate-json-articles.js +0 -20
- package/dist/find-duplicate-json-articles.js.map +1 -1
- package/dist/infrastructures/articles-command-repository.d.ts +40 -0
- package/dist/infrastructures/articles-command-repository.d.ts.map +1 -0
- package/dist/infrastructures/articles-command-repository.js +40 -0
- package/dist/infrastructures/articles-command-repository.js.map +1 -0
- package/dist/infrastructures/books-command-repository.d.ts +17 -0
- package/dist/infrastructures/books-command-repository.d.ts.map +1 -0
- package/dist/infrastructures/books-command-repository.js +25 -0
- package/dist/infrastructures/books-command-repository.js.map +1 -0
- package/dist/infrastructures/images-command-repository.d.ts +17 -0
- package/dist/infrastructures/images-command-repository.d.ts.map +1 -0
- package/dist/infrastructures/images-command-repository.js +25 -0
- package/dist/infrastructures/images-command-repository.js.map +1 -0
- package/dist/infrastructures/notes-command-repository.d.ts +17 -0
- package/dist/infrastructures/notes-command-repository.d.ts.map +1 -0
- package/dist/infrastructures/notes-command-repository.js +25 -0
- package/dist/infrastructures/notes-command-repository.js.map +1 -0
- package/dist/rag/chunker.d.ts +10 -0
- package/dist/rag/chunker.d.ts.map +1 -0
- package/dist/rag/chunker.js +188 -0
- package/dist/rag/chunker.js.map +1 -0
- package/dist/rag/config.d.ts +44 -0
- package/dist/rag/config.d.ts.map +1 -0
- package/dist/rag/config.js +34 -0
- package/dist/rag/config.js.map +1 -0
- package/dist/rag/embedding.d.ts +15 -0
- package/dist/rag/embedding.d.ts.map +1 -0
- package/dist/rag/embedding.js +61 -0
- package/dist/rag/embedding.js.map +1 -0
- package/dist/rag/ingest.d.ts +3 -0
- package/dist/rag/ingest.d.ts.map +1 -0
- package/dist/rag/ingest.js +148 -0
- package/dist/rag/ingest.js.map +1 -0
- package/dist/rag/qdrant-client.d.ts +40 -0
- package/dist/rag/qdrant-client.d.ts.map +1 -0
- package/dist/rag/qdrant-client.js +160 -0
- package/dist/rag/qdrant-client.js.map +1 -0
- package/dist/rag/search.d.ts +3 -0
- package/dist/rag/search.d.ts.map +1 -0
- package/dist/rag/search.js +105 -0
- package/dist/rag/search.js.map +1 -0
- package/dist/reset-articles.js +10 -16
- package/dist/reset-articles.js.map +1 -1
- package/dist/reset-books.js +8 -17
- package/dist/reset-books.js.map +1 -1
- package/dist/reset-images.js +8 -17
- package/dist/reset-images.js.map +1 -1
- package/dist/reset-notes.js +8 -17
- package/dist/reset-notes.js.map +1 -1
- package/dist/revert-articles.js +9 -8
- package/dist/revert-articles.js.map +1 -1
- package/dist/revert-books.js +7 -9
- package/dist/revert-books.js.map +1 -1
- package/dist/revert-images.js +7 -9
- package/dist/revert-images.js.map +1 -1
- package/dist/revert-notes.js +7 -9
- package/dist/revert-notes.js.map +1 -1
- package/dist/update-raw-articles.js +1 -1
- package/dist/update-raw-articles.js.map +1 -1
- package/package.json +11 -4
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
-
import { createPushoverService } from "@s-hirano-ist/s-notification";
|
|
5
4
|
function getAllJsonFiles(directory) {
|
|
6
5
|
return fs
|
|
7
6
|
.readdirSync(directory)
|
|
@@ -100,33 +99,14 @@ function displayResults(duplicates) {
|
|
|
100
99
|
});
|
|
101
100
|
}
|
|
102
101
|
async function main() {
|
|
103
|
-
const env = {
|
|
104
|
-
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
105
|
-
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
106
|
-
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
107
|
-
};
|
|
108
|
-
if (Object.values(env).some((v) => !v)) {
|
|
109
|
-
throw new Error("Required environment variables are not set.");
|
|
110
|
-
}
|
|
111
|
-
const notificationService = createPushoverService({
|
|
112
|
-
url: env.PUSHOVER_URL ?? "",
|
|
113
|
-
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
114
|
-
appToken: env.PUSHOVER_APP_TOKEN ?? "",
|
|
115
|
-
});
|
|
116
102
|
try {
|
|
117
103
|
console.log("json/articleディレクトリ内のURL重複を検索中...\n");
|
|
118
104
|
const duplicates = findDuplicateUrls();
|
|
119
105
|
displayResults(duplicates);
|
|
120
106
|
console.log("\n検索完了");
|
|
121
|
-
await notificationService.notifyInfo("find-duplicate-json-articles completed", {
|
|
122
|
-
caller: "find-duplicate-json-articles",
|
|
123
|
-
});
|
|
124
107
|
}
|
|
125
108
|
catch (error) {
|
|
126
109
|
console.error("❌ エラーが発生しました:", error);
|
|
127
|
-
await notificationService.notifyError(`find-duplicate-json-articles failed: ${error}`, {
|
|
128
|
-
caller: "find-duplicate-json-articles",
|
|
129
|
-
});
|
|
130
110
|
process.exit(1);
|
|
131
111
|
}
|
|
132
112
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-duplicate-json-articles.js","sourceRoot":"","sources":["../src/find-duplicate-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"find-duplicate-json-articles.js","sourceRoot":"","sources":["../src/find-duplicate-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA6BjC,SAAS,eAAe,CAAC,SAAiB;IACzC,OAAO,EAAE;SACP,WAAW,CAAC,SAAS,CAAC;SACtB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,WAAW,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,eAAe,CAAC,QAAgB;IACxC,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAgB,CAAC;AAChD,CAAC;AAED,SAAS,iBAAiB;IACzB,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAC;IAEpD,OAAO,CAAC,GAAG,CAAC,mBAAmB,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEjD,mBAAmB;IACnB,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,eAAe;IACf,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAElD,IAAI,CAAC;YACJ,MAAM,WAAW,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC;YAErC,OAAO,CAAC,GAAG,CACV,QAAQ,QAAQ,WAAW,QAAQ,UAAU,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CACvE,CAAC;YAEF,aAAa;YACb,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,CAAC;gBACrC,SAAS,EAAE,CAAC;gBAEZ,MAAM,UAAU,GAAkB;oBACjC,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,QAAQ;oBACR,QAAQ;iBACR,CAAC;gBAEF,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;gBACxC,CAAC;qBAAM,CAAC;oBACP,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;gBACpC,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,QAAQ,QAAQ,gBAAgB,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,EAAE,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAExC,WAAW;IACX,MAAM,UAAU,GAAmB,EAAE,CAAC;IAEtC,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACf,GAAG;gBACH,WAAW;aACX,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,OAAO,UAAU,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,UAA0B;IACjD,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,WAAW,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;IAE9C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACR,CAAC;IAED,eAAe;IACf,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAEvE,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE;QACvC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,UAAU,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,WAAW,SAAS,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC;QAExD,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE;YACtD,OAAO,CAAC,GAAG,CACV,MAAM,QAAQ,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,KAAK,UAAU,CAAC,QAAQ,EAAE,CACrE,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK;IACvB,CAAC,CAAC,CAAC;IAEH,SAAS;IACT,MAAM,yBAAyB,GAAG,UAAU,CAAC,MAAM,CAClD,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,WAAW,CAAC,MAAM,EAC1C,CAAC,CACD,CAAC;IACF,MAAM,aAAa,GAAG,yBAAyB,GAAG,UAAU,CAAC,MAAM,CAAC;IAEpE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CACV,kBAAkB,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,IAAI,CAAC,KAAK,CAC7D,CAAC;IAEF,aAAa;IACb,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC1B,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,aAAa,CAAC,GAAG,CAChB,GAAG,CAAC,QAAQ,EACZ,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAC1C,CAAC;QACH,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;SACjC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE;QAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,KAAK,KAAK,UAAU,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,IAAI,CAAC;QACJ,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAElD,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC;QACvC,cAAc,CAAC,UAAU,CAAC,CAAC;QAE3B,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { IBatchCommandRepository } from "@s-hirano-ist/s-core/common";
|
|
2
|
+
/**
|
|
3
|
+
* Prisma client type for transaction support.
|
|
4
|
+
*
|
|
5
|
+
* @remarks
|
|
6
|
+
* This is a minimal type that supports both regular PrismaClient
|
|
7
|
+
* and transaction clients (from $transaction).
|
|
8
|
+
* Uses a generic approach to accept Prisma's strict typing.
|
|
9
|
+
*/
|
|
10
|
+
type PrismaClientLike = {
|
|
11
|
+
article: {
|
|
12
|
+
updateMany: (args: any) => Promise<{
|
|
13
|
+
count: number;
|
|
14
|
+
}>;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Creates an Articles batch command repository for bulk operations.
|
|
19
|
+
*
|
|
20
|
+
* @param prisma - The Prisma client or transaction client
|
|
21
|
+
* @returns An IBatchCommandRepository implementation for articles
|
|
22
|
+
*
|
|
23
|
+
* @remarks
|
|
24
|
+
* This is a specialized implementation for batch scripts.
|
|
25
|
+
* It only implements bulkUpdateStatus for performance.
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* ```typescript
|
|
29
|
+
* const prisma = new PrismaClient({ accelerateUrl: env.DATABASE_URL });
|
|
30
|
+
*
|
|
31
|
+
* await prisma.$transaction(async (tx) => {
|
|
32
|
+
* const repo = createArticlesCommandRepository(tx);
|
|
33
|
+
* const batchService = new ArticlesBatchDomainService(repo);
|
|
34
|
+
* await batchService.resetArticles(userId);
|
|
35
|
+
* });
|
|
36
|
+
* ```
|
|
37
|
+
*/
|
|
38
|
+
export declare function createArticlesCommandRepository(prisma: PrismaClientLike): IBatchCommandRepository;
|
|
39
|
+
export {};
|
|
40
|
+
//# sourceMappingURL=articles-command-repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"articles-command-repository.d.ts","sourceRoot":"","sources":["../../src/infrastructures/articles-command-repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,uBAAuB,EAEvB,MAAM,6BAA6B,CAAC;AAErC;;;;;;;GAOG;AACH,KAAK,gBAAgB,GAAG;IACvB,OAAO,EAAE;QACR,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtD,CAAC;CACF,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,+BAA+B,CAC9C,MAAM,EAAE,gBAAgB,GACtB,uBAAuB,CAqBzB"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates an Articles batch command repository for bulk operations.
|
|
3
|
+
*
|
|
4
|
+
* @param prisma - The Prisma client or transaction client
|
|
5
|
+
* @returns An IBatchCommandRepository implementation for articles
|
|
6
|
+
*
|
|
7
|
+
* @remarks
|
|
8
|
+
* This is a specialized implementation for batch scripts.
|
|
9
|
+
* It only implements bulkUpdateStatus for performance.
|
|
10
|
+
*
|
|
11
|
+
* @example
|
|
12
|
+
* ```typescript
|
|
13
|
+
* const prisma = new PrismaClient({ accelerateUrl: env.DATABASE_URL });
|
|
14
|
+
*
|
|
15
|
+
* await prisma.$transaction(async (tx) => {
|
|
16
|
+
* const repo = createArticlesCommandRepository(tx);
|
|
17
|
+
* const batchService = new ArticlesBatchDomainService(repo);
|
|
18
|
+
* await batchService.resetArticles(userId);
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export function createArticlesCommandRepository(prisma) {
|
|
23
|
+
return {
|
|
24
|
+
async bulkUpdateStatus(params) {
|
|
25
|
+
const { userId, fromStatus, toStatus, exportedAt } = params;
|
|
26
|
+
const result = await prisma.article.updateMany({
|
|
27
|
+
where: {
|
|
28
|
+
userId: userId,
|
|
29
|
+
status: fromStatus,
|
|
30
|
+
},
|
|
31
|
+
data: {
|
|
32
|
+
status: toStatus,
|
|
33
|
+
...(exportedAt && { exportedAt: exportedAt }),
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
return { count: result.count };
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=articles-command-repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"articles-command-repository.js","sourceRoot":"","sources":["../../src/infrastructures/articles-command-repository.ts"],"names":[],"mappings":"AAoBA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,+BAA+B,CAC9C,MAAwB;IAExB,OAAO;QACN,KAAK,CAAC,gBAAgB,CACrB,MAA8B;YAE9B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC9C,KAAK,EAAE;oBACN,MAAM,EAAE,MAAgB;oBACxB,MAAM,EAAE,UAAoB;iBAC5B;gBACD,IAAI,EAAE;oBACL,MAAM,EAAE,QAAkB;oBAC1B,GAAG,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,UAAkB,EAAE,CAAC;iBACrD;aACD,CAAC,CAAC;YAEH,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { IBatchCommandRepository } from "@s-hirano-ist/s-core/common";
|
|
2
|
+
type PrismaClientLike = {
|
|
3
|
+
book: {
|
|
4
|
+
updateMany: (args: any) => Promise<{
|
|
5
|
+
count: number;
|
|
6
|
+
}>;
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Creates a Books batch command repository for bulk operations.
|
|
11
|
+
*
|
|
12
|
+
* @param prisma - The Prisma client or transaction client
|
|
13
|
+
* @returns An IBatchCommandRepository implementation for books
|
|
14
|
+
*/
|
|
15
|
+
export declare function createBooksCommandRepository(prisma: PrismaClientLike): IBatchCommandRepository;
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=books-command-repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"books-command-repository.d.ts","sourceRoot":"","sources":["../../src/infrastructures/books-command-repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,uBAAuB,EAEvB,MAAM,6BAA6B,CAAC;AAErC,KAAK,gBAAgB,GAAG;IACvB,IAAI,EAAE;QAEL,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtD,CAAC;CACF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC3C,MAAM,EAAE,gBAAgB,GACtB,uBAAuB,CAqBzB"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a Books batch command repository for bulk operations.
|
|
3
|
+
*
|
|
4
|
+
* @param prisma - The Prisma client or transaction client
|
|
5
|
+
* @returns An IBatchCommandRepository implementation for books
|
|
6
|
+
*/
|
|
7
|
+
export function createBooksCommandRepository(prisma) {
|
|
8
|
+
return {
|
|
9
|
+
async bulkUpdateStatus(params) {
|
|
10
|
+
const { userId, fromStatus, toStatus, exportedAt } = params;
|
|
11
|
+
const result = await prisma.book.updateMany({
|
|
12
|
+
where: {
|
|
13
|
+
userId: userId,
|
|
14
|
+
status: fromStatus,
|
|
15
|
+
},
|
|
16
|
+
data: {
|
|
17
|
+
status: toStatus,
|
|
18
|
+
...(exportedAt && { exportedAt: exportedAt }),
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
return { count: result.count };
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=books-command-repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"books-command-repository.js","sourceRoot":"","sources":["../../src/infrastructures/books-command-repository.ts"],"names":[],"mappings":"AAaA;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAC3C,MAAwB;IAExB,OAAO;QACN,KAAK,CAAC,gBAAgB,CACrB,MAA8B;YAE9B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC3C,KAAK,EAAE;oBACN,MAAM,EAAE,MAAgB;oBACxB,MAAM,EAAE,UAAoB;iBAC5B;gBACD,IAAI,EAAE;oBACL,MAAM,EAAE,QAAkB;oBAC1B,GAAG,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,UAAkB,EAAE,CAAC;iBACrD;aACD,CAAC,CAAC;YAEH,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { IBatchCommandRepository } from "@s-hirano-ist/s-core/common";
|
|
2
|
+
type PrismaClientLike = {
|
|
3
|
+
image: {
|
|
4
|
+
updateMany: (args: any) => Promise<{
|
|
5
|
+
count: number;
|
|
6
|
+
}>;
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Creates an Images batch command repository for bulk operations.
|
|
11
|
+
*
|
|
12
|
+
* @param prisma - The Prisma client or transaction client
|
|
13
|
+
* @returns An IBatchCommandRepository implementation for images
|
|
14
|
+
*/
|
|
15
|
+
export declare function createImagesCommandRepository(prisma: PrismaClientLike): IBatchCommandRepository;
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=images-command-repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images-command-repository.d.ts","sourceRoot":"","sources":["../../src/infrastructures/images-command-repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,uBAAuB,EAEvB,MAAM,6BAA6B,CAAC;AAErC,KAAK,gBAAgB,GAAG;IACvB,KAAK,EAAE;QAEN,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtD,CAAC;CACF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAC5C,MAAM,EAAE,gBAAgB,GACtB,uBAAuB,CAqBzB"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates an Images batch command repository for bulk operations.
|
|
3
|
+
*
|
|
4
|
+
* @param prisma - The Prisma client or transaction client
|
|
5
|
+
* @returns An IBatchCommandRepository implementation for images
|
|
6
|
+
*/
|
|
7
|
+
export function createImagesCommandRepository(prisma) {
|
|
8
|
+
return {
|
|
9
|
+
async bulkUpdateStatus(params) {
|
|
10
|
+
const { userId, fromStatus, toStatus, exportedAt } = params;
|
|
11
|
+
const result = await prisma.image.updateMany({
|
|
12
|
+
where: {
|
|
13
|
+
userId: userId,
|
|
14
|
+
status: fromStatus,
|
|
15
|
+
},
|
|
16
|
+
data: {
|
|
17
|
+
status: toStatus,
|
|
18
|
+
...(exportedAt && { exportedAt: exportedAt }),
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
return { count: result.count };
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=images-command-repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images-command-repository.js","sourceRoot":"","sources":["../../src/infrastructures/images-command-repository.ts"],"names":[],"mappings":"AAaA;;;;;GAKG;AACH,MAAM,UAAU,6BAA6B,CAC5C,MAAwB;IAExB,OAAO;QACN,KAAK,CAAC,gBAAgB,CACrB,MAA8B;YAE9B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;gBAC5C,KAAK,EAAE;oBACN,MAAM,EAAE,MAAgB;oBACxB,MAAM,EAAE,UAAoB;iBAC5B;gBACD,IAAI,EAAE;oBACL,MAAM,EAAE,QAAkB;oBAC1B,GAAG,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,UAAkB,EAAE,CAAC;iBACrD;aACD,CAAC,CAAC;YAEH,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { IBatchCommandRepository } from "@s-hirano-ist/s-core/common";
|
|
2
|
+
type PrismaClientLike = {
|
|
3
|
+
note: {
|
|
4
|
+
updateMany: (args: any) => Promise<{
|
|
5
|
+
count: number;
|
|
6
|
+
}>;
|
|
7
|
+
};
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Creates a Notes batch command repository for bulk operations.
|
|
11
|
+
*
|
|
12
|
+
* @param prisma - The Prisma client or transaction client
|
|
13
|
+
* @returns An IBatchCommandRepository implementation for notes
|
|
14
|
+
*/
|
|
15
|
+
export declare function createNotesCommandRepository(prisma: PrismaClientLike): IBatchCommandRepository;
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=notes-command-repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"notes-command-repository.d.ts","sourceRoot":"","sources":["../../src/infrastructures/notes-command-repository.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEX,uBAAuB,EAEvB,MAAM,6BAA6B,CAAC;AAErC,KAAK,gBAAgB,GAAG;IACvB,IAAI,EAAE;QAEL,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,OAAO,CAAC;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KACtD,CAAC;CACF,CAAC;AAEF;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAC3C,MAAM,EAAE,gBAAgB,GACtB,uBAAuB,CAqBzB"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a Notes batch command repository for bulk operations.
|
|
3
|
+
*
|
|
4
|
+
* @param prisma - The Prisma client or transaction client
|
|
5
|
+
* @returns An IBatchCommandRepository implementation for notes
|
|
6
|
+
*/
|
|
7
|
+
export function createNotesCommandRepository(prisma) {
|
|
8
|
+
return {
|
|
9
|
+
async bulkUpdateStatus(params) {
|
|
10
|
+
const { userId, fromStatus, toStatus, exportedAt } = params;
|
|
11
|
+
const result = await prisma.note.updateMany({
|
|
12
|
+
where: {
|
|
13
|
+
userId: userId,
|
|
14
|
+
status: fromStatus,
|
|
15
|
+
},
|
|
16
|
+
data: {
|
|
17
|
+
status: toStatus,
|
|
18
|
+
...(exportedAt && { exportedAt: exportedAt }),
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
return { count: result.count };
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=notes-command-repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"notes-command-repository.js","sourceRoot":"","sources":["../../src/infrastructures/notes-command-repository.ts"],"names":[],"mappings":"AAaA;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAC3C,MAAwB;IAExB,OAAO;QACN,KAAK,CAAC,gBAAgB,CACrB,MAA8B;YAE9B,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;YAE5D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC3C,KAAK,EAAE;oBACN,MAAM,EAAE,MAAgB;oBACxB,MAAM,EAAE,UAAoB;iBAC5B;gBACD,IAAI,EAAE;oBACL,MAAM,EAAE,QAAkB;oBAC1B,GAAG,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,UAAkB,EAAE,CAAC;iBACrD;aACD,CAAC,CAAC;YAEH,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAChC,CAAC;KACD,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type QdrantPayload } from "./config.js";
|
|
2
|
+
/**
|
|
3
|
+
* Parse JSON article file and generate chunks
|
|
4
|
+
*/
|
|
5
|
+
export declare function parseJsonArticle(filePath: string, content: string): QdrantPayload[];
|
|
6
|
+
/**
|
|
7
|
+
* Parse Markdown file and generate chunks
|
|
8
|
+
*/
|
|
9
|
+
export declare function parseMarkdown(filePath: string, content: string): QdrantPayload[];
|
|
10
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,aAAa,EAAc,MAAM,aAAa,CAAC;AAsC7D;;GAEG;AACH,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACb,aAAa,EAAE,CAsCjB;AAiID;;GAEG;AACH,wBAAgB,aAAa,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,GACb,aAAa,EAAE,CA2CjB"}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { createHash } from "crypto";
|
|
2
|
+
import { RAG_CONFIG } from "./config.js";
|
|
3
|
+
/**
|
|
4
|
+
* Generate content hash for change detection
|
|
5
|
+
*/
|
|
6
|
+
function generateHash(content) {
|
|
7
|
+
return createHash("sha256").update(content).digest("hex").slice(0, 16);
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Generate chunk ID from doc_id and index
|
|
11
|
+
*/
|
|
12
|
+
function generateChunkId(docId, index) {
|
|
13
|
+
return `${docId}#${index}`;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Parse JSON article file and generate chunks
|
|
17
|
+
*/
|
|
18
|
+
export function parseJsonArticle(filePath, content) {
|
|
19
|
+
const json = JSON.parse(content);
|
|
20
|
+
const docId = `file:${filePath}`;
|
|
21
|
+
const chunks = [];
|
|
22
|
+
for (let i = 0; i < json.body.length; i++) {
|
|
23
|
+
const item = json.body[i];
|
|
24
|
+
// Build text from available fields
|
|
25
|
+
const textParts = [];
|
|
26
|
+
if (item.title)
|
|
27
|
+
textParts.push(item.title);
|
|
28
|
+
if (item.ogTitle && item.ogTitle !== item.title)
|
|
29
|
+
textParts.push(item.ogTitle);
|
|
30
|
+
if (item.ogDescription)
|
|
31
|
+
textParts.push(item.ogDescription);
|
|
32
|
+
if (item.quote)
|
|
33
|
+
textParts.push(item.quote);
|
|
34
|
+
if (item.url)
|
|
35
|
+
textParts.push(item.url);
|
|
36
|
+
const text = textParts.join("\n");
|
|
37
|
+
// Skip empty items
|
|
38
|
+
if (!text.trim())
|
|
39
|
+
continue;
|
|
40
|
+
const chunkId = generateChunkId(docId, i);
|
|
41
|
+
chunks.push({
|
|
42
|
+
type: "bookmark_json",
|
|
43
|
+
top_heading: json.heading,
|
|
44
|
+
doc_id: docId,
|
|
45
|
+
chunk_id: chunkId,
|
|
46
|
+
title: item.title || item.ogTitle || "Untitled",
|
|
47
|
+
url: item.url,
|
|
48
|
+
heading_path: [json.heading],
|
|
49
|
+
text,
|
|
50
|
+
content_hash: generateHash(text),
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
return chunks;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Parse Markdown frontmatter
|
|
57
|
+
*/
|
|
58
|
+
function parseFrontmatter(content) {
|
|
59
|
+
const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
|
60
|
+
if (!frontmatterMatch) {
|
|
61
|
+
return {
|
|
62
|
+
frontmatter: { heading: "unknown" },
|
|
63
|
+
body: content,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
const frontmatterStr = frontmatterMatch[1];
|
|
67
|
+
const body = frontmatterMatch[2];
|
|
68
|
+
// Simple YAML parsing for our needs
|
|
69
|
+
const frontmatter = { heading: "unknown" };
|
|
70
|
+
for (const line of frontmatterStr.split("\n")) {
|
|
71
|
+
const [key, ...valueParts] = line.split(":");
|
|
72
|
+
const value = valueParts.join(":").trim();
|
|
73
|
+
if (key === "heading") {
|
|
74
|
+
frontmatter.heading = value;
|
|
75
|
+
}
|
|
76
|
+
else if (key === "description") {
|
|
77
|
+
frontmatter.description = value;
|
|
78
|
+
}
|
|
79
|
+
else if (key === "draft") {
|
|
80
|
+
frontmatter.draft = value === "true";
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return { frontmatter, body };
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Split markdown into sections by headings
|
|
87
|
+
*/
|
|
88
|
+
function splitMarkdownByHeadings(content) {
|
|
89
|
+
const lines = content.split("\n");
|
|
90
|
+
const sections = [];
|
|
91
|
+
let currentHeadingPath = [];
|
|
92
|
+
let currentSection = null;
|
|
93
|
+
const headingStack = [];
|
|
94
|
+
for (const line of lines) {
|
|
95
|
+
const headingMatch = line.match(/^(#{2,3})\s+(.+)$/);
|
|
96
|
+
if (headingMatch) {
|
|
97
|
+
// Save previous section
|
|
98
|
+
if (currentSection && currentSection.content.trim()) {
|
|
99
|
+
sections.push(currentSection);
|
|
100
|
+
}
|
|
101
|
+
const level = headingMatch[1].length;
|
|
102
|
+
const title = headingMatch[2];
|
|
103
|
+
// Update heading stack
|
|
104
|
+
while (headingStack.length > 0 &&
|
|
105
|
+
headingStack[headingStack.length - 1].level >= level) {
|
|
106
|
+
headingStack.pop();
|
|
107
|
+
}
|
|
108
|
+
headingStack.push({ level, title });
|
|
109
|
+
// Update heading path
|
|
110
|
+
currentHeadingPath = headingStack.map((h) => h.title);
|
|
111
|
+
currentSection = {
|
|
112
|
+
headingPath: [...currentHeadingPath],
|
|
113
|
+
title,
|
|
114
|
+
content: "",
|
|
115
|
+
level,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
else if (currentSection) {
|
|
119
|
+
currentSection.content += line + "\n";
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
// Save last section
|
|
123
|
+
if (currentSection && currentSection.content.trim()) {
|
|
124
|
+
sections.push(currentSection);
|
|
125
|
+
}
|
|
126
|
+
return sections;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Split long text into smaller chunks by paragraphs
|
|
130
|
+
*/
|
|
131
|
+
function splitByParagraphs(text, maxLength) {
|
|
132
|
+
if (text.length <= maxLength) {
|
|
133
|
+
return [text];
|
|
134
|
+
}
|
|
135
|
+
const paragraphs = text.split(/\n\n+/);
|
|
136
|
+
const chunks = [];
|
|
137
|
+
let currentChunk = "";
|
|
138
|
+
for (const para of paragraphs) {
|
|
139
|
+
if (currentChunk.length + para.length > maxLength && currentChunk) {
|
|
140
|
+
chunks.push(currentChunk.trim());
|
|
141
|
+
currentChunk = para;
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
currentChunk += (currentChunk ? "\n\n" : "") + para;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
if (currentChunk.trim()) {
|
|
148
|
+
chunks.push(currentChunk.trim());
|
|
149
|
+
}
|
|
150
|
+
return chunks;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Parse Markdown file and generate chunks
|
|
154
|
+
*/
|
|
155
|
+
export function parseMarkdown(filePath, content) {
|
|
156
|
+
const { frontmatter, body } = parseFrontmatter(content);
|
|
157
|
+
const docId = `file:${filePath}`;
|
|
158
|
+
const chunks = [];
|
|
159
|
+
// Skip draft files
|
|
160
|
+
if (frontmatter.draft) {
|
|
161
|
+
return [];
|
|
162
|
+
}
|
|
163
|
+
const sections = splitMarkdownByHeadings(body);
|
|
164
|
+
let chunkIndex = 0;
|
|
165
|
+
for (const section of sections) {
|
|
166
|
+
// Split long sections
|
|
167
|
+
const textChunks = splitByParagraphs(section.content, RAG_CONFIG.chunking.maxChunkLength);
|
|
168
|
+
for (const text of textChunks) {
|
|
169
|
+
if (!text.trim())
|
|
170
|
+
continue;
|
|
171
|
+
const fullHeadingPath = [frontmatter.heading, ...section.headingPath];
|
|
172
|
+
const chunkId = generateChunkId(docId, chunkIndex);
|
|
173
|
+
chunks.push({
|
|
174
|
+
type: "markdown_note",
|
|
175
|
+
top_heading: frontmatter.heading,
|
|
176
|
+
doc_id: docId,
|
|
177
|
+
chunk_id: chunkId,
|
|
178
|
+
title: section.title,
|
|
179
|
+
heading_path: fullHeadingPath,
|
|
180
|
+
text,
|
|
181
|
+
content_hash: generateHash(text),
|
|
182
|
+
});
|
|
183
|
+
chunkIndex++;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return chunks;
|
|
187
|
+
}
|
|
188
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAsB,UAAU,EAAE,MAAM,aAAa,CAAC;AAwB7D;;GAEG;AACH,SAAS,YAAY,CAAC,OAAe;IACpC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACxE,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAAa,EAAE,KAAa;IACpD,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC/B,QAAgB,EAChB,OAAe;IAEf,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAgB,CAAC;IAChD,MAAM,KAAK,GAAG,QAAQ,QAAQ,EAAE,CAAC;IACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE1B,mCAAmC;QACnC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK;YAC9C,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,aAAa;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,GAAG;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAElC,mBAAmB;QACnB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAS;QAE3B,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAE1C,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,IAAI,CAAC,OAAO;YACzB,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,IAAI,UAAU;YAC/C,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,YAAY,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI;YACJ,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;SAChC,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IAIxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IAE5E,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACvB,OAAO;YACN,WAAW,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE;YACnC,IAAI,EAAE,OAAO;SACb,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAEjC,oCAAoC;IACpC,MAAM,WAAW,GAAwB,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAEhE,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,WAAW,CAAC,OAAO,GAAG,KAAK,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;YAClC,WAAW,CAAC,WAAW,GAAG,KAAK,CAAC;QACjC,CAAC;aAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC5B,WAAW,CAAC,KAAK,GAAG,KAAK,KAAK,MAAM,CAAC;QACtC,CAAC;IACF,CAAC;IAED,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAC9B,CAAC;AASD;;GAEG;AACH,SAAS,uBAAuB,CAAC,OAAe;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IAEvC,IAAI,kBAAkB,GAAa,EAAE,CAAC;IACtC,IAAI,cAAc,GAA2B,IAAI,CAAC;IAClD,MAAM,YAAY,GAAuC,EAAE,CAAC;IAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAErD,IAAI,YAAY,EAAE,CAAC;YAClB,wBAAwB;YACxB,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;gBACrD,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC/B,CAAC;YAED,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YAE9B,uBAAuB;YACvB,OACC,YAAY,CAAC,MAAM,GAAG,CAAC;gBACvB,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,KAAK,EACnD,CAAC;gBACF,YAAY,CAAC,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAEpC,sBAAsB;YACtB,kBAAkB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAEtD,cAAc,GAAG;gBAChB,WAAW,EAAE,CAAC,GAAG,kBAAkB,CAAC;gBACpC,KAAK;gBACL,OAAO,EAAE,EAAE;gBACX,KAAK;aACL,CAAC;QACH,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC3B,cAAc,CAAC,OAAO,IAAI,IAAI,GAAG,IAAI,CAAC;QACvC,CAAC;IACF,CAAC;IAED,oBAAoB;IACpB,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC/B,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAY,EAAE,SAAiB;IACzD,IAAI,IAAI,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,EAAE,CAAC;IAEtB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;YACnE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YACjC,YAAY,GAAG,IAAI,CAAC;QACrB,CAAC;aAAM,CAAC;YACP,YAAY,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;QACrD,CAAC;IACF,CAAC;IAED,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAC5B,QAAgB,EAChB,OAAe;IAEf,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,QAAQ,QAAQ,EAAE,CAAC;IACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;IAEnC,mBAAmB;IACnB,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACvB,OAAO,EAAE,CAAC;IACX,CAAC;IAED,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,sBAAsB;QACtB,MAAM,UAAU,GAAG,iBAAiB,CACnC,OAAO,CAAC,OAAO,EACf,UAAU,CAAC,QAAQ,CAAC,cAAc,CAClC,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAS;YAE3B,MAAM,eAAe,GAAG,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,eAAe;gBACrB,WAAW,EAAE,WAAW,CAAC,OAAO;gBAChC,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,OAAO;gBACjB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,YAAY,EAAE,eAAe;gBAC7B,IAAI;gBACJ,YAAY,EAAE,YAAY,CAAC,IAAI,CAAC;aAChC,CAAC,CAAC;YAEH,UAAU,EAAE,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export declare const RAG_CONFIG: {
|
|
2
|
+
readonly qdrant: {
|
|
3
|
+
readonly collectionName: "knowledge_v1";
|
|
4
|
+
readonly vectorSize: 384;
|
|
5
|
+
readonly distance: "Cosine";
|
|
6
|
+
};
|
|
7
|
+
readonly embedding: {
|
|
8
|
+
readonly model: "intfloat/multilingual-e5-small";
|
|
9
|
+
readonly prefix: {
|
|
10
|
+
readonly query: "query: ";
|
|
11
|
+
readonly passage: "passage: ";
|
|
12
|
+
};
|
|
13
|
+
};
|
|
14
|
+
readonly paths: {
|
|
15
|
+
readonly markdown: readonly ["markdown/note/**/*.md", "markdown/book/**/*.md", "raw/article/**/*.md"];
|
|
16
|
+
readonly json: "json/article/**/*.json";
|
|
17
|
+
};
|
|
18
|
+
readonly chunking: {
|
|
19
|
+
readonly maxChunkLength: 2000;
|
|
20
|
+
readonly headingLevels: readonly [2, 3];
|
|
21
|
+
};
|
|
22
|
+
readonly hashCachePath: ".rag-hash-cache.json";
|
|
23
|
+
};
|
|
24
|
+
export type QdrantPayload = {
|
|
25
|
+
type: "markdown_note" | "bookmark_json";
|
|
26
|
+
top_heading: string;
|
|
27
|
+
doc_id: string;
|
|
28
|
+
chunk_id: string;
|
|
29
|
+
title: string;
|
|
30
|
+
url?: string;
|
|
31
|
+
heading_path: string[];
|
|
32
|
+
text: string;
|
|
33
|
+
content_hash: string;
|
|
34
|
+
};
|
|
35
|
+
export type SearchResult = {
|
|
36
|
+
score: number;
|
|
37
|
+
text: string;
|
|
38
|
+
title: string;
|
|
39
|
+
url?: string;
|
|
40
|
+
heading_path: string[];
|
|
41
|
+
type: "markdown_note" | "bookmark_json";
|
|
42
|
+
doc_id: string;
|
|
43
|
+
};
|
|
44
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/rag/config.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,UAAU;;;;;;;;;;;;;;;;;;;;;;CAmCb,CAAC;AAGX,MAAM,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC;IACxC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;CACrB,CAAC;AAGF,MAAM,MAAM,YAAY,GAAG;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,EAAE,eAAe,GAAG,eAAe,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC;CACf,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// RAG Configuration
|
|
2
|
+
export const RAG_CONFIG = {
|
|
3
|
+
// Qdrant settings
|
|
4
|
+
qdrant: {
|
|
5
|
+
collectionName: "knowledge_v1",
|
|
6
|
+
vectorSize: 384, // multilingual-e5-small
|
|
7
|
+
distance: "Cosine",
|
|
8
|
+
},
|
|
9
|
+
// Embedding settings
|
|
10
|
+
embedding: {
|
|
11
|
+
model: "intfloat/multilingual-e5-small",
|
|
12
|
+
prefix: {
|
|
13
|
+
query: "query: ",
|
|
14
|
+
passage: "passage: ",
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
// File paths
|
|
18
|
+
paths: {
|
|
19
|
+
markdown: [
|
|
20
|
+
"markdown/note/**/*.md",
|
|
21
|
+
"markdown/book/**/*.md",
|
|
22
|
+
"raw/article/**/*.md",
|
|
23
|
+
],
|
|
24
|
+
json: "json/article/**/*.json",
|
|
25
|
+
},
|
|
26
|
+
// Chunking settings
|
|
27
|
+
chunking: {
|
|
28
|
+
maxChunkLength: 2000,
|
|
29
|
+
headingLevels: [2, 3], // ## and ###
|
|
30
|
+
},
|
|
31
|
+
// Cache file for hash comparison
|
|
32
|
+
hashCachePath: ".rag-hash-cache.json",
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/rag/config.ts"],"names":[],"mappings":"AAAA,oBAAoB;AACpB,MAAM,CAAC,MAAM,UAAU,GAAG;IACzB,kBAAkB;IAClB,MAAM,EAAE;QACP,cAAc,EAAE,cAAc;QAC9B,UAAU,EAAE,GAAG,EAAE,wBAAwB;QACzC,QAAQ,EAAE,QAAiB;KAC3B;IAED,qBAAqB;IACrB,SAAS,EAAE;QACV,KAAK,EAAE,gCAAgC;QACvC,MAAM,EAAE;YACP,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,WAAW;SACpB;KACD;IAED,aAAa;IACb,KAAK,EAAE;QACN,QAAQ,EAAE;YACT,uBAAuB;YACvB,uBAAuB;YACvB,qBAAqB;SACrB;QACD,IAAI,EAAE,wBAAwB;KAC9B;IAED,oBAAoB;IACpB,QAAQ,EAAE;QACT,cAAc,EAAE,IAAI;QACpB,aAAa,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa;KACpC;IAED,iCAAiC;IACjC,aAAa,EAAE,sBAAsB;CAC5B,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generate embedding for a single text
|
|
3
|
+
* @param text - Input text to embed
|
|
4
|
+
* @param isQuery - Whether this is a query (vs passage)
|
|
5
|
+
* @returns Embedding vector
|
|
6
|
+
*/
|
|
7
|
+
export declare function embed(text: string, isQuery?: boolean): Promise<number[]>;
|
|
8
|
+
/**
|
|
9
|
+
* Generate embeddings for multiple texts in batch
|
|
10
|
+
* @param texts - Array of input texts
|
|
11
|
+
* @param isQuery - Whether these are queries (vs passages)
|
|
12
|
+
* @returns Array of embedding vectors
|
|
13
|
+
*/
|
|
14
|
+
export declare function embedBatch(texts: string[], isQuery?: boolean): Promise<number[][]>;
|
|
15
|
+
//# sourceMappingURL=embedding.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding.d.ts","sourceRoot":"","sources":["../../src/rag/embedding.ts"],"names":[],"mappings":"AAwBA;;;;;GAKG;AACH,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,UAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiB5E;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAC/B,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,UAAQ,GACb,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA2BrB"}
|