@s-hirano-ist/s-scripts 1.18.6 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fix-book-frontmatter.d.ts +3 -0
- package/dist/fix-book-frontmatter.d.ts.map +1 -0
- package/dist/fix-book-frontmatter.js +70 -0
- package/dist/fix-book-frontmatter.js.map +1 -0
- package/dist/fix-image-formats.d.ts +3 -0
- package/dist/fix-image-formats.d.ts.map +1 -0
- package/dist/fix-image-formats.js +82 -0
- package/dist/fix-image-formats.js.map +1 -0
- package/dist/rag/ingest.js +3 -21
- package/dist/rag/ingest.js.map +1 -1
- package/dist/rag/search.js +2 -16
- package/dist/rag/search.js.map +1 -1
- package/dist/validate-json-articles.d.ts +3 -0
- package/dist/validate-json-articles.d.ts.map +1 -0
- package/dist/validate-json-articles.js +147 -0
- package/dist/validate-json-articles.js.map +1 -0
- package/package.json +13 -7
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix-book-frontmatter.d.ts","sourceRoot":"","sources":["../src/fix-book-frontmatter.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { basename, join } from "node:path";
|
|
4
|
+
import { glob } from "glob";
|
|
5
|
+
function hasFrontmatter(content) {
|
|
6
|
+
return content.startsWith("---");
|
|
7
|
+
}
|
|
8
|
+
function extractTitleFromH1(content) {
|
|
9
|
+
const match = content.match(/^# (.+)$/m);
|
|
10
|
+
return match ? match[1].trim() : null;
|
|
11
|
+
}
|
|
12
|
+
function generateFrontmatter(isbn, title) {
|
|
13
|
+
return [
|
|
14
|
+
"---",
|
|
15
|
+
`heading: ${isbn}`,
|
|
16
|
+
`description: ${title}`,
|
|
17
|
+
"draft: false",
|
|
18
|
+
"---",
|
|
19
|
+
].join("\n");
|
|
20
|
+
}
|
|
21
|
+
async function main() {
|
|
22
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
23
|
+
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
24
|
+
const bookDir = join(contentsPath, "markdown", "book");
|
|
25
|
+
console.log(`markdown/book ディレクトリのfrontmatterを検証中...${dryRun ? " (dry-run)" : ""}\n`);
|
|
26
|
+
const files = await glob(`${bookDir}/*.md`);
|
|
27
|
+
console.log(`検出ファイル数: ${files.length}\n`);
|
|
28
|
+
let fixedCount = 0;
|
|
29
|
+
let skippedCount = 0;
|
|
30
|
+
let errorCount = 0;
|
|
31
|
+
for (const filePath of files) {
|
|
32
|
+
const fileName = basename(filePath);
|
|
33
|
+
const isbn = basename(filePath, ".md");
|
|
34
|
+
try {
|
|
35
|
+
const content = await readFile(filePath, "utf8");
|
|
36
|
+
if (hasFrontmatter(content)) {
|
|
37
|
+
skippedCount++;
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const title = extractTitleFromH1(content);
|
|
41
|
+
if (!title) {
|
|
42
|
+
console.error(`⚠️ ${fileName}: H1タイトルが見つかりません。スキップします。`);
|
|
43
|
+
errorCount++;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const frontmatter = generateFrontmatter(isbn, title);
|
|
47
|
+
const newContent = `${frontmatter}\n\n${content}`;
|
|
48
|
+
if (dryRun) {
|
|
49
|
+
console.log(`🔍 [dry-run] 修正予定: ${fileName}`);
|
|
50
|
+
console.log(` heading: ${isbn}`);
|
|
51
|
+
console.log(` description: ${title}`);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
await writeFile(filePath, newContent, "utf8");
|
|
55
|
+
console.log(`✅ 修正完了: ${fileName} (${title})`);
|
|
56
|
+
}
|
|
57
|
+
fixedCount++;
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
console.error(`❌ ${fileName}: ${error}`);
|
|
61
|
+
errorCount++;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
console.log(`\n📊 結果: 修正 ${fixedCount} 件, スキップ ${skippedCount} 件, エラー ${errorCount} 件${dryRun ? " (dry-run)" : ""}`);
|
|
65
|
+
}
|
|
66
|
+
main().catch((error) => {
|
|
67
|
+
console.error("❌ エラーが発生しました:", error);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
});
|
|
70
|
+
//# sourceMappingURL=fix-book-frontmatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix-book-frontmatter.js","sourceRoot":"","sources":["../src/fix-book-frontmatter.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,SAAS,cAAc,CAAC,OAAe;IACtC,OAAO,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe;IAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,KAAa;IACvD,OAAO;QACN,KAAK;QACL,YAAY,IAAI,EAAE;QAClB,gBAAgB,KAAK,EAAE;QACvB,cAAc;QACd,KAAK;KACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACd,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAEvD,OAAO,CAAC,GAAG,CACV,0CAA0C,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,CACxE,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAE1C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEvC,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEjD,IAAI,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC7B,YAAY,EAAE,CAAC;gBACf,SAAS;YACV,CAAC;YAED,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CACZ,MAAM,QAAQ,2BAA2B,CACzC,CAAC;gBACF,UAAU,EAAE,CAAC;gBACb,SAAS;YACV,CAAC;YAED,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACrD,MAAM,UAAU,GAAG,GAAG,WAAW,OAAO,OAAO,EAAE,CAAC;YAElD,IAAI,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,EAAE,CAAC,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACP,MAAM,SAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,KAAK,KAAK,GAAG,CAAC,CAAC;YAC/C,CAAC;YACD,UAAU,EAAE,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YACzC,UAAU,EAAE,CAAC;QACd,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CACV,eAAe,UAAU,YAAY,YAAY,WAAW,UAAU,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CACvG,CAAC;AACH,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix-image-formats.d.ts","sourceRoot":"","sources":["../src/fix-image-formats.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { stat, unlink } from "node:fs/promises";
|
|
3
|
+
import { basename, extname, join } from "node:path";
|
|
4
|
+
import { glob } from "glob";
|
|
5
|
+
import sharp from "sharp";
|
|
6
|
+
const WEBP_QUALITY = 80;
|
|
7
|
+
function isConvertibleImage(filePath) {
|
|
8
|
+
const ext = extname(filePath).toLowerCase();
|
|
9
|
+
return [".jpg", ".jpeg", ".png"].includes(ext);
|
|
10
|
+
}
|
|
11
|
+
function getWebpOutputPath(filePath) {
|
|
12
|
+
const ext = extname(filePath);
|
|
13
|
+
return `${filePath.slice(0, -ext.length)}.webp`;
|
|
14
|
+
}
|
|
15
|
+
async function convertToWebp(inputPath, outputPath) {
|
|
16
|
+
await sharp(inputPath).webp({ quality: WEBP_QUALITY }).toFile(outputPath);
|
|
17
|
+
}
|
|
18
|
+
async function processDirectory(directory, label, dryRun) {
|
|
19
|
+
const files = await glob(`${directory}/*`);
|
|
20
|
+
const convertible = files.filter(isConvertibleImage);
|
|
21
|
+
console.log(`\n📁 ${label}: ${convertible.length} 件の変換対象を検出（全 ${files.length} ファイル中）`);
|
|
22
|
+
let converted = 0;
|
|
23
|
+
let skipped = 0;
|
|
24
|
+
let errors = 0;
|
|
25
|
+
for (const filePath of convertible) {
|
|
26
|
+
const fileName = basename(filePath);
|
|
27
|
+
const outputPath = getWebpOutputPath(filePath);
|
|
28
|
+
try {
|
|
29
|
+
// 出力先が既に存在する場合はスキップ
|
|
30
|
+
try {
|
|
31
|
+
await stat(outputPath);
|
|
32
|
+
console.log(`⏭️ スキップ（WebP既存）: ${fileName}`);
|
|
33
|
+
skipped++;
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// ファイルが存在しない場合は変換続行
|
|
38
|
+
}
|
|
39
|
+
if (dryRun) {
|
|
40
|
+
console.log(`🔍 [dry-run] 変換予定: ${fileName} → ${basename(outputPath)}`);
|
|
41
|
+
converted++;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
await convertToWebp(filePath, outputPath);
|
|
45
|
+
// 出力ファイルの存在とサイズを確認
|
|
46
|
+
const outputStat = await stat(outputPath);
|
|
47
|
+
if (outputStat.size === 0) {
|
|
48
|
+
throw new Error("変換後のファイルサイズが0です");
|
|
49
|
+
}
|
|
50
|
+
await unlink(filePath);
|
|
51
|
+
console.log(`✅ 変換完了: ${fileName} → ${basename(outputPath)}`);
|
|
52
|
+
converted++;
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
console.error(`❌ ${fileName}: ${error}`);
|
|
56
|
+
errors++;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return { converted, skipped, errors };
|
|
60
|
+
}
|
|
61
|
+
async function main() {
|
|
62
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
63
|
+
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
64
|
+
console.log(`画像をWebPフォーマットに変換中...${dryRun ? " (dry-run)" : ""}`);
|
|
65
|
+
const bookDir = join(contentsPath, "image", "book");
|
|
66
|
+
const dumpDir = join(contentsPath, "image", "dump");
|
|
67
|
+
console.log("📌 image/note/ はSVGファイルのためスキップします。");
|
|
68
|
+
const bookResult = await processDirectory(bookDir, "image/book", dryRun);
|
|
69
|
+
const dumpResult = await processDirectory(dumpDir, "image/dump", dryRun);
|
|
70
|
+
const totalConverted = bookResult.converted + dumpResult.converted;
|
|
71
|
+
const totalSkipped = bookResult.skipped + dumpResult.skipped;
|
|
72
|
+
const totalErrors = bookResult.errors + dumpResult.errors;
|
|
73
|
+
console.log(`\n📊 合計: 変換 ${totalConverted} 件, スキップ ${totalSkipped} 件, エラー ${totalErrors} 件${dryRun ? " (dry-run)" : ""}`);
|
|
74
|
+
if (totalErrors > 0) {
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
main().catch((error) => {
|
|
79
|
+
console.error("❌ エラーが発生しました:", error);
|
|
80
|
+
process.exit(1);
|
|
81
|
+
});
|
|
82
|
+
//# sourceMappingURL=fix-image-formats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fix-image-formats.js","sourceRoot":"","sources":["../src/fix-image-formats.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB,SAAS,kBAAkB,CAAC,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC9B,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAC3B,SAAiB,EACjB,UAAkB;IAElB,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC9B,SAAiB,EACjB,KAAa,EACb,MAAe;IAEf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,SAAS,IAAI,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAErD,OAAO,CAAC,GAAG,CACV,QAAQ,KAAK,KAAK,WAAW,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CACzE,CAAC;IAEF,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,QAAQ,IAAI,WAAW,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,UAAU,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QAE/C,IAAI,CAAC;YACJ,oBAAoB;YACpB,IAAI,CAAC;gBACJ,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;gBAC5C,OAAO,EAAE,CAAC;gBACV,SAAS;YACV,CAAC;YAAC,MAAM,CAAC;gBACR,oBAAoB;YACrB,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CACV,sBAAsB,QAAQ,MAAM,QAAQ,CAAC,UAAU,CAAC,EAAE,CAC1D,CAAC;gBACF,SAAS,EAAE,CAAC;gBACZ,SAAS;YACV,CAAC;YAED,MAAM,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAE1C,mBAAmB;YACnB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1C,IAAI,UAAU,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;YACpC,CAAC;YAED,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,WAAW,QAAQ,MAAM,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC7D,SAAS,EAAE,CAAC;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,KAAK,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;YACzC,MAAM,EAAE,CAAC;QACV,CAAC;IACF,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAElE,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAEpD,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAElD,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAEzE,MAAM,cAAc,GAAG,UAAU,CAAC,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC;IACnE,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IAC7D,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAE1D,OAAO,CAAC,GAAG,CACV,eAAe,cAAc,YAAY,YAAY,WAAW,WAAW,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5G,CAAC;IAEF,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/rag/ingest.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
3
|
import { parseJsonArticle, parseMarkdown, } from "@s-hirano-ist/s-search/chunker";
|
|
4
|
-
import { createEmbeddingClient } from "@s-hirano-ist/s-search/embedding-client";
|
|
5
4
|
import { ingestChunks } from "@s-hirano-ist/s-search/ingest";
|
|
6
5
|
import { ensureCollection, getCollectionStats, } from "@s-hirano-ist/s-search/qdrant-client";
|
|
7
6
|
import { glob } from "glob";
|
|
@@ -72,34 +71,17 @@ async function ingest(force) {
|
|
|
72
71
|
console.error(`Error parsing ${file.path}:`, error);
|
|
73
72
|
}
|
|
74
73
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
cfAccessClientId: process.env.CF_ACCESS_CLIENT_ID ?? "",
|
|
78
|
-
cfAccessClientSecret: process.env.CF_ACCESS_CLIENT_SECRET ?? "",
|
|
79
|
-
}).embedBatch;
|
|
80
|
-
// Delegate to core ingest logic
|
|
81
|
-
const result = await ingestChunks(allChunks, { embedBatchFn, force });
|
|
74
|
+
// Delegate to core ingest logic (Qdrant Inference handles embedding)
|
|
75
|
+
const result = await ingestChunks(allChunks, { force });
|
|
82
76
|
// Get final stats
|
|
83
77
|
const finalStats = await getCollectionStats();
|
|
84
78
|
console.log(`\nFinal points count: ${finalStats.pointsCount}`);
|
|
85
79
|
console.log(`Ingest completed successfully! (${result.changedChunks} changed, ${result.skippedChunks} skipped)`);
|
|
86
80
|
}
|
|
87
81
|
async function main() {
|
|
88
|
-
|
|
89
|
-
QDRANT_URL: process.env.QDRANT_URL,
|
|
90
|
-
EMBEDDING_API_URL: process.env.EMBEDDING_API_URL,
|
|
91
|
-
CF_ACCESS_CLIENT_ID: process.env.CF_ACCESS_CLIENT_ID,
|
|
92
|
-
CF_ACCESS_CLIENT_SECRET: process.env.CF_ACCESS_CLIENT_SECRET,
|
|
93
|
-
};
|
|
94
|
-
if (!env.QDRANT_URL) {
|
|
82
|
+
if (!process.env.QDRANT_URL) {
|
|
95
83
|
throw new Error("QDRANT_URL environment variable is required.");
|
|
96
84
|
}
|
|
97
|
-
if (!env.EMBEDDING_API_URL) {
|
|
98
|
-
throw new Error("EMBEDDING_API_URL environment variable is required.");
|
|
99
|
-
}
|
|
100
|
-
if (!env.CF_ACCESS_CLIENT_ID || !env.CF_ACCESS_CLIENT_SECRET) {
|
|
101
|
-
throw new Error("CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET environment variables are required.");
|
|
102
|
-
}
|
|
103
85
|
const force = process.argv.includes("--force");
|
|
104
86
|
try {
|
|
105
87
|
await ingest(force);
|
package/dist/rag/ingest.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/rag/ingest.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EACN,gBAAgB,EAChB,aAAa,GACb,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAC7D,OAAO,EACN,gBAAgB,EAChB,kBAAkB,GAClB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAQnD;;GAEG;AACH,KAAK,UAAU,SAAS;IACvB,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,kBAAkB;IAClB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC9D,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACxD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,IAAc;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAEjD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,MAAM,CAAC,KAAc;IACnC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAEpC,2BAA2B;IAC3B,MAAM,gBAAgB,EAAE,CAAC;IAEzB,oBAAoB;IACpB,MAAM,YAAY,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,yBAAyB,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;IAEnE,IAAI,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAChE,CAAC;IAED,iBAAiB;IACjB,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,iBAAiB,YAAY,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,IAAI,CAAC,CAAC;IAEzC,8BAA8B;IAC9B,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAChC,MAAM,SAAS,GAAoB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QAC3B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACF,CAAC;IAED,qEAAqE;IACrE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAExD,kBAAkB;IAClB,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CACV,mCAAmC,MAAM,CAAC,aAAa,aAAa,MAAM,CAAC,aAAa,WAAW,CACnG,CAAC;AACH,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAE/C,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/rag/search.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { createEmbeddingClient } from "@s-hirano-ist/s-search/embedding-client";
|
|
3
2
|
import { ensureCollection, search as qdrantSearch, } from "@s-hirano-ist/s-search/qdrant-client";
|
|
4
3
|
function parseArgs() {
|
|
5
4
|
const args = process.argv.slice(2);
|
|
@@ -32,14 +31,8 @@ function parseArgs() {
|
|
|
32
31
|
}
|
|
33
32
|
return { query, options };
|
|
34
33
|
}
|
|
35
|
-
const embeddingClient = createEmbeddingClient({
|
|
36
|
-
apiUrl: process.env.EMBEDDING_API_URL ?? "",
|
|
37
|
-
cfAccessClientId: process.env.CF_ACCESS_CLIENT_ID ?? "",
|
|
38
|
-
cfAccessClientSecret: process.env.CF_ACCESS_CLIENT_SECRET ?? "",
|
|
39
|
-
});
|
|
40
34
|
async function searchContent(query, options = {}) {
|
|
41
|
-
const
|
|
42
|
-
const results = await qdrantSearch(queryVector, {
|
|
35
|
+
const results = await qdrantSearch(query, {
|
|
43
36
|
topK: options.topK,
|
|
44
37
|
filter: {
|
|
45
38
|
type: options.type,
|
|
@@ -76,16 +69,9 @@ async function search() {
|
|
|
76
69
|
}
|
|
77
70
|
}
|
|
78
71
|
async function main() {
|
|
79
|
-
|
|
80
|
-
QDRANT_URL: process.env.QDRANT_URL,
|
|
81
|
-
EMBEDDING_API_URL: process.env.EMBEDDING_API_URL,
|
|
82
|
-
};
|
|
83
|
-
if (!env.QDRANT_URL) {
|
|
72
|
+
if (!process.env.QDRANT_URL) {
|
|
84
73
|
throw new Error("QDRANT_URL environment variable is required.");
|
|
85
74
|
}
|
|
86
|
-
if (!env.EMBEDDING_API_URL) {
|
|
87
|
-
throw new Error("EMBEDDING_API_URL environment variable is required.");
|
|
88
|
-
}
|
|
89
75
|
try {
|
|
90
76
|
await search();
|
|
91
77
|
}
|
package/dist/rag/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/rag/search.ts"],"names":[],"mappings":";AAEA,OAAO,
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/rag/search.ts"],"names":[],"mappings":";AAEA,OAAO,EACN,gBAAgB,EAChB,MAAM,IAAI,YAAY,GACtB,MAAM,sCAAsC,CAAC;AAS9C,SAAS,SAAS;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAA0B,CAAC;QACnD,CAAC;aAAM,IAAI,GAAG,KAAK,WAAW,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACvD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,GAAG,KAAK,gBAAgB,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACxB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAkB,CAAC;YAChD,OAAO,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC7D,CAAC;aAAM,CAAC;YACP,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CACZ,sIAAsI,CACtI,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,aAAa,CAC3B,KAAa,EACb,UAAyB,EAAE;IAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE;QACzC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE;YACP,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,WAAW,EAAE,OAAO,CAAC,OAAO;YAC5B,YAAY,EAAE,OAAO,CAAC,WAAW;SACjC;KACD,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,MAAM;IACpB,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;IAEvC,OAAO,CAAC,GAAG,CAAC,mBAAmB,KAAK,GAAG,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,IAAI,OAAO,CAAC,WAAW;QACtB,OAAO,CAAC,GAAG,CACV,mBAAmB,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAC7G,CAAC;IACH,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,MAAM,gBAAgB,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAErD,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,CAAC,YAAY,aAAa,CAAC,CAAC;IAEzD,KAAK,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACtC,IAAI,MAAM,CAAC,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,SAAS,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;IACnC,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,CAAC;QACJ,MAAM,MAAM,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-json-articles.d.ts","sourceRoot":"","sources":["../src/validate-json-articles.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
function validateSchema(fileName, data) {
|
|
5
|
+
const issues = [];
|
|
6
|
+
if (typeof data !== "object" || data === null || Array.isArray(data)) {
|
|
7
|
+
issues.push({
|
|
8
|
+
file: fileName,
|
|
9
|
+
level: "error",
|
|
10
|
+
message: "トップレベルがオブジェクトではありません",
|
|
11
|
+
});
|
|
12
|
+
return issues;
|
|
13
|
+
}
|
|
14
|
+
const obj = data;
|
|
15
|
+
if (typeof obj.heading !== "string" || obj.heading.length === 0) {
|
|
16
|
+
issues.push({
|
|
17
|
+
file: fileName,
|
|
18
|
+
level: "error",
|
|
19
|
+
message: "heading が存在しないか空文字です",
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
if (typeof obj.description !== "string" || obj.description.length === 0) {
|
|
23
|
+
issues.push({
|
|
24
|
+
file: fileName,
|
|
25
|
+
level: "error",
|
|
26
|
+
message: "description が存在しないか空文字です",
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
if (!Array.isArray(obj.body)) {
|
|
30
|
+
issues.push({
|
|
31
|
+
file: fileName,
|
|
32
|
+
level: "error",
|
|
33
|
+
message: "body が配列ではありません",
|
|
34
|
+
});
|
|
35
|
+
return issues;
|
|
36
|
+
}
|
|
37
|
+
for (let i = 0; i < obj.body.length; i++) {
|
|
38
|
+
const item = obj.body[i];
|
|
39
|
+
if (typeof item.title !== "string" || item.title.length === 0) {
|
|
40
|
+
issues.push({
|
|
41
|
+
file: fileName,
|
|
42
|
+
level: "error",
|
|
43
|
+
message: `body[${i}]: title が存在しないか空文字です`,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
if (typeof item.url !== "string" || item.url.length === 0) {
|
|
47
|
+
issues.push({
|
|
48
|
+
file: fileName,
|
|
49
|
+
level: "error",
|
|
50
|
+
message: `body[${i}]: url が存在しないか空文字です`,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return issues;
|
|
55
|
+
}
|
|
56
|
+
function checkDescriptionFixme(fileName, data) {
|
|
57
|
+
if (data.description === "FIXME") {
|
|
58
|
+
return [
|
|
59
|
+
{
|
|
60
|
+
file: fileName,
|
|
61
|
+
level: "warning",
|
|
62
|
+
message: `description が "FIXME" です（heading: "${data.heading}"）`,
|
|
63
|
+
},
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
68
|
+
function displayResults(issues) {
|
|
69
|
+
const errors = issues.filter((i) => i.level === "error");
|
|
70
|
+
const warnings = issues.filter((i) => i.level === "warning");
|
|
71
|
+
console.log("\n=== JSON Article 検証結果 ===");
|
|
72
|
+
console.log(`エラー: ${errors.length} 件, 警告: ${warnings.length} 件\n`);
|
|
73
|
+
if (issues.length === 0) {
|
|
74
|
+
console.log("✅ 問題は見つかりませんでした！");
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
// ファイル別にグループ化
|
|
78
|
+
const byFile = new Map();
|
|
79
|
+
for (const issue of issues) {
|
|
80
|
+
const existing = byFile.get(issue.file) ?? [];
|
|
81
|
+
existing.push(issue);
|
|
82
|
+
byFile.set(issue.file, existing);
|
|
83
|
+
}
|
|
84
|
+
for (const [file, fileIssues] of byFile.entries()) {
|
|
85
|
+
console.log(`📄 ${file}`);
|
|
86
|
+
for (const issue of fileIssues) {
|
|
87
|
+
const icon = issue.level === "error" ? "❌" : "⚠️";
|
|
88
|
+
console.log(` ${icon} ${issue.message}`);
|
|
89
|
+
}
|
|
90
|
+
console.log("");
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async function main() {
|
|
94
|
+
const contentsPath = process.env.S_CONTENTS_PATH ?? process.cwd();
|
|
95
|
+
const articleDirectory = join(contentsPath, "json", "article");
|
|
96
|
+
console.log("json/article ディレクトリのJSONファイルを検証中...\n");
|
|
97
|
+
const jsonFiles = fs
|
|
98
|
+
.readdirSync(articleDirectory)
|
|
99
|
+
.filter((file) => file.endsWith(".json") && file !== ".DS_Store");
|
|
100
|
+
console.log(`検証対象ファイル数: ${jsonFiles.length}\n`);
|
|
101
|
+
const allIssues = [];
|
|
102
|
+
let totalArticles = 0;
|
|
103
|
+
for (const fileName of jsonFiles) {
|
|
104
|
+
const filePath = join(articleDirectory, fileName);
|
|
105
|
+
try {
|
|
106
|
+
const content = fs.readFileSync(filePath, "utf8");
|
|
107
|
+
const data = JSON.parse(content);
|
|
108
|
+
allIssues.push(...validateSchema(fileName, data));
|
|
109
|
+
if (typeof data === "object" &&
|
|
110
|
+
data !== null &&
|
|
111
|
+
!Array.isArray(data) &&
|
|
112
|
+
Array.isArray(data.body)) {
|
|
113
|
+
const articlesJson = data;
|
|
114
|
+
totalArticles += articlesJson.body.length;
|
|
115
|
+
allIssues.push(...checkDescriptionFixme(fileName, articlesJson));
|
|
116
|
+
console.log(`処理中: ${fileName} (${articlesJson.heading}, ${articlesJson.body.length} 件)`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (error instanceof SyntaxError) {
|
|
121
|
+
allIssues.push({
|
|
122
|
+
file: fileName,
|
|
123
|
+
level: "error",
|
|
124
|
+
message: `JSON解析エラー: ${error.message}`,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
allIssues.push({
|
|
129
|
+
file: fileName,
|
|
130
|
+
level: "error",
|
|
131
|
+
message: `ファイル読み込みエラー: ${error}`,
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
console.log(`\n合計記事数: ${totalArticles}`);
|
|
137
|
+
displayResults(allIssues);
|
|
138
|
+
const errorCount = allIssues.filter((i) => i.level === "error").length;
|
|
139
|
+
if (errorCount > 0) {
|
|
140
|
+
process.exit(1);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
main().catch((error) => {
|
|
144
|
+
console.error("❌ エラーが発生しました:", error);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
});
|
|
147
|
+
//# sourceMappingURL=validate-json-articles.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-json-articles.js","sourceRoot":"","sources":["../src/validate-json-articles.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAwBjC,SAAS,cAAc,CAAC,QAAgB,EAAE,IAAa;IACtD,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,sBAAsB;SAC/B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,IAA+B,CAAC;IAE5C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,sBAAsB;SAC/B,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzE,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,0BAA0B;SACnC,CAAC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,iBAAiB;SAC1B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC;QAEpD,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/D,MAAM,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,QAAQ,CAAC,uBAAuB;aACzC,CAAC,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,OAAO;gBACd,OAAO,EAAE,QAAQ,CAAC,qBAAqB;aACvC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,qBAAqB,CAC7B,QAAgB,EAChB,IAAkB;IAElB,IAAI,IAAI,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;QAClC,OAAO;YACN;gBACC,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,SAAS;gBAChB,OAAO,EAAE,sCAAsC,IAAI,CAAC,OAAO,IAAI;aAC/D;SACD,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACX,CAAC;AAED,SAAS,cAAc,CAAC,MAAyB;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAE7D,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAC3C,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,MAAM,WAAW,QAAQ,CAAC,MAAM,MAAM,CAAC,CAAC;IAEnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,OAAO;IACR,CAAC;IAED,cAAc;IACd,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6B,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QAC1B,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;YAClD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,KAAK,UAAU,IAAI;IAClB,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAClE,MAAM,gBAAgB,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IAE/D,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,EAAE;SAClB,WAAW,CAAC,gBAAgB,CAAC;SAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,KAAK,WAAW,CAAC,CAAC;IAEnE,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,EAAE,QAAQ,CAAC,CAAC;QAElD,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAE1C,SAAS,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YAElD,IACC,OAAO,IAAI,KAAK,QAAQ;gBACxB,IAAI,KAAK,IAAI;gBACb,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBACpB,KAAK,CAAC,OAAO,CAAE,IAAqB,CAAC,IAAI,CAAC,EACzC,CAAC;gBACF,MAAM,YAAY,GAAG,IAAoB,CAAC;gBAC1C,aAAa,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC1C,SAAS,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC;gBAEjE,OAAO,CAAC,GAAG,CACV,QAAQ,QAAQ,KAAK,YAAY,CAAC,OAAO,KAAK,YAAY,CAAC,IAAI,CAAC,MAAM,KAAK,CAC3E,CAAC;YACH,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;gBAClC,SAAS,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,cAAc,KAAK,CAAC,OAAO,EAAE;iBACtC,CAAC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACP,SAAS,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,gBAAgB,KAAK,EAAE;iBAChC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,aAAa,EAAE,CAAC,CAAC;IAEzC,cAAc,CAAC,SAAS,CAAC,CAAC;IAE1B,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACvE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;AACF,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACtB,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@s-hirano-ist/s-scripts",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.19.0",
|
|
4
4
|
"description": "CLI scripts for s-private data operations",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -26,7 +26,10 @@
|
|
|
26
26
|
"ingest-books": "./dist/ingest-books.js",
|
|
27
27
|
"ingest-articles": "./dist/ingest-articles.js",
|
|
28
28
|
"ingest-images": "./dist/ingest-images.js",
|
|
29
|
-
"cleanup-minio-images": "./dist/cleanup-minio-images.js"
|
|
29
|
+
"cleanup-minio-images": "./dist/cleanup-minio-images.js",
|
|
30
|
+
"fix-image-formats": "./dist/fix-image-formats.js",
|
|
31
|
+
"fix-book-frontmatter": "./dist/fix-book-frontmatter.js",
|
|
32
|
+
"validate-json-articles": "./dist/validate-json-articles.js"
|
|
30
33
|
},
|
|
31
34
|
"files": [
|
|
32
35
|
"dist",
|
|
@@ -48,11 +51,11 @@
|
|
|
48
51
|
"iconv-lite": "0.7.2",
|
|
49
52
|
"sharp": "0.34.5",
|
|
50
53
|
"turndown": "7.2.4",
|
|
51
|
-
"@s-hirano-ist/s-core": "1.18.
|
|
54
|
+
"@s-hirano-ist/s-core": "1.18.7",
|
|
55
|
+
"@s-hirano-ist/s-database": "1.18.6",
|
|
52
56
|
"@s-hirano-ist/s-notification": "1.18.2",
|
|
53
|
-
"@s-hirano-ist/s-search": "1.18.
|
|
54
|
-
"@s-hirano-ist/s-storage": "1.18.4"
|
|
55
|
-
"@s-hirano-ist/s-database": "1.18.5"
|
|
57
|
+
"@s-hirano-ist/s-search": "1.18.7",
|
|
58
|
+
"@s-hirano-ist/s-storage": "1.18.4"
|
|
56
59
|
},
|
|
57
60
|
"devDependencies": {
|
|
58
61
|
"@types/node": "24.12.2",
|
|
@@ -86,6 +89,9 @@
|
|
|
86
89
|
"ingest-books": "tsx src/ingest-books.ts",
|
|
87
90
|
"ingest-articles": "tsx src/ingest-articles.ts",
|
|
88
91
|
"ingest-images": "tsx src/ingest-images.ts",
|
|
89
|
-
"cleanup-minio-images": "tsx src/cleanup-minio-images.ts"
|
|
92
|
+
"cleanup-minio-images": "tsx src/cleanup-minio-images.ts",
|
|
93
|
+
"fix-image-formats": "tsx src/fix-image-formats.ts",
|
|
94
|
+
"fix-book-frontmatter": "tsx src/fix-book-frontmatter.ts",
|
|
95
|
+
"validate-json-articles": "tsx src/validate-json-articles.ts"
|
|
90
96
|
}
|
|
91
97
|
}
|