@s-hirano-ist/s-scripts 1.26.8 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/chunk-5OXUZW2Q.js +590 -0
- package/dist/chunk-7PVU2Z32.js +71 -0
- package/dist/chunk-AJMTRCPJ.js +15 -0
- package/dist/chunk-AZF2R2BF.js +85 -0
- package/dist/chunk-F4J6JBR4.js +115 -0
- package/dist/chunk-HDYQV2Y2.js +58 -0
- package/dist/chunk-HIK7SX34.js +85 -0
- package/dist/chunk-IWNZQXDG.js +7660 -0
- package/dist/chunk-J4VLH4VA.js +267 -0
- package/dist/chunk-JZOR5GA2.js +42 -0
- package/dist/chunk-KKZP7ZF3.js +77 -0
- package/dist/chunk-N6TZF755.js +102 -0
- package/dist/chunk-S5WJ5VXW.js +85 -0
- package/dist/chunk-UXGLXHQ6.js +40 -0
- package/dist/cleanup-minio-images.js +115 -92
- package/dist/enrich-books.js +87 -78
- package/dist/fetch-articles.js +101 -90
- package/dist/fetch-books.js +147 -128
- package/dist/fetch-images.js +88 -72
- package/dist/fetch-notes.js +66 -54
- package/dist/find-duplicate-json-articles.js +100 -97
- package/dist/fix-book-frontmatter.js +111 -99
- package/dist/fix-image-formats.js +81 -65
- package/dist/ingest-articles.js +30 -249
- package/dist/ingest-books.js +61 -318
- package/dist/ingest-images.js +33 -184
- package/dist/ingest-notes.js +41 -171
- package/dist/rag/delete-collection.js +32 -22
- package/dist/rag/ingest.js +371 -92
- package/dist/rag/search.js +80 -76
- package/dist/reset-articles.js +57 -60
- package/dist/reset-books.js +57 -59
- package/dist/reset-images.js +57 -59
- package/dist/reset-notes.js +57 -59
- package/dist/revert-articles.js +53 -55
- package/dist/revert-books.js +53 -53
- package/dist/revert-images.js +53 -53
- package/dist/revert-notes.js +53 -53
- package/dist/s-content-format.js +103 -13
- package/dist/update-json-articles.js +101 -97
- package/dist/update-raw-articles.js +130 -130
- package/dist/validate-json-articles.js +129 -130
- package/package.json +13 -50
- package/dist/cleanup-minio-images.d.ts +0 -3
- package/dist/cleanup-minio-images.d.ts.map +0 -1
- package/dist/cleanup-minio-images.js.map +0 -1
- package/dist/enrich-books.d.ts +0 -3
- package/dist/enrich-books.d.ts.map +0 -1
- package/dist/enrich-books.js.map +0 -1
- package/dist/fetch-articles.d.ts +0 -3
- package/dist/fetch-articles.d.ts.map +0 -1
- package/dist/fetch-articles.js.map +0 -1
- package/dist/fetch-books.d.ts +0 -3
- package/dist/fetch-books.d.ts.map +0 -1
- package/dist/fetch-books.js.map +0 -1
- package/dist/fetch-images.d.ts +0 -3
- package/dist/fetch-images.d.ts.map +0 -1
- package/dist/fetch-images.js.map +0 -1
- package/dist/fetch-notes.d.ts +0 -3
- package/dist/fetch-notes.d.ts.map +0 -1
- package/dist/fetch-notes.js.map +0 -1
- package/dist/find-duplicate-json-articles.d.ts +0 -3
- package/dist/find-duplicate-json-articles.d.ts.map +0 -1
- package/dist/find-duplicate-json-articles.js.map +0 -1
- package/dist/fix-book-frontmatter.d.ts +0 -3
- package/dist/fix-book-frontmatter.d.ts.map +0 -1
- package/dist/fix-book-frontmatter.js.map +0 -1
- package/dist/fix-image-formats.d.ts +0 -3
- package/dist/fix-image-formats.d.ts.map +0 -1
- package/dist/fix-image-formats.js.map +0 -1
- package/dist/infrastructures/articles-command-repository.d.ts +0 -57
- package/dist/infrastructures/articles-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/articles-command-repository.js +0 -68
- package/dist/infrastructures/articles-command-repository.js.map +0 -1
- package/dist/infrastructures/books-command-repository.d.ts +0 -34
- package/dist/infrastructures/books-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/books-command-repository.js +0 -53
- package/dist/infrastructures/books-command-repository.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.d.ts +0 -9
- package/dist/infrastructures/cache-invalidation-client.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.js +0 -21
- package/dist/infrastructures/cache-invalidation-client.js.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.d.ts +0 -2
- package/dist/infrastructures/cache-invalidation-client.test.d.ts.map +0 -1
- package/dist/infrastructures/cache-invalidation-client.test.js +0 -25
- package/dist/infrastructures/cache-invalidation-client.test.js.map +0 -1
- package/dist/infrastructures/images-command-repository.d.ts +0 -34
- package/dist/infrastructures/images-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/images-command-repository.js +0 -53
- package/dist/infrastructures/images-command-repository.js.map +0 -1
- package/dist/infrastructures/notes-command-repository.d.ts +0 -34
- package/dist/infrastructures/notes-command-repository.d.ts.map +0 -1
- package/dist/infrastructures/notes-command-repository.js +0 -53
- package/dist/infrastructures/notes-command-repository.js.map +0 -1
- package/dist/ingest-articles.d.ts +0 -3
- package/dist/ingest-articles.d.ts.map +0 -1
- package/dist/ingest-articles.js.map +0 -1
- package/dist/ingest-books.d.ts +0 -3
- package/dist/ingest-books.d.ts.map +0 -1
- package/dist/ingest-books.js.map +0 -1
- package/dist/ingest-images.d.ts +0 -3
- package/dist/ingest-images.d.ts.map +0 -1
- package/dist/ingest-images.js.map +0 -1
- package/dist/ingest-notes.d.ts +0 -3
- package/dist/ingest-notes.d.ts.map +0 -1
- package/dist/ingest-notes.js.map +0 -1
- package/dist/lib/glob-paths.d.ts +0 -6
- package/dist/lib/glob-paths.d.ts.map +0 -1
- package/dist/lib/glob-paths.js +0 -9
- package/dist/lib/glob-paths.js.map +0 -1
- package/dist/lib/glob-paths.test.d.ts +0 -2
- package/dist/lib/glob-paths.test.d.ts.map +0 -1
- package/dist/lib/glob-paths.test.js +0 -30
- package/dist/lib/glob-paths.test.js.map +0 -1
- package/dist/lib/html-charset.d.ts +0 -4
- package/dist/lib/html-charset.d.ts.map +0 -1
- package/dist/lib/html-charset.js +0 -33
- package/dist/lib/html-charset.js.map +0 -1
- package/dist/lib/html-charset.test.d.ts +0 -2
- package/dist/lib/html-charset.test.d.ts.map +0 -1
- package/dist/lib/html-charset.test.js +0 -27
- package/dist/lib/html-charset.test.js.map +0 -1
- package/dist/rag/delete-collection.d.ts +0 -3
- package/dist/rag/delete-collection.d.ts.map +0 -1
- package/dist/rag/delete-collection.js.map +0 -1
- package/dist/rag/ingest-config.d.ts +0 -8
- package/dist/rag/ingest-config.d.ts.map +0 -1
- package/dist/rag/ingest-config.js +0 -8
- package/dist/rag/ingest-config.js.map +0 -1
- package/dist/rag/ingest.d.ts +0 -3
- package/dist/rag/ingest.d.ts.map +0 -1
- package/dist/rag/ingest.js.map +0 -1
- package/dist/rag/search.d.ts +0 -3
- package/dist/rag/search.d.ts.map +0 -1
- package/dist/rag/search.js.map +0 -1
- package/dist/reset-articles.d.ts +0 -3
- package/dist/reset-articles.d.ts.map +0 -1
- package/dist/reset-articles.js.map +0 -1
- package/dist/reset-books.d.ts +0 -3
- package/dist/reset-books.d.ts.map +0 -1
- package/dist/reset-books.js.map +0 -1
- package/dist/reset-images.d.ts +0 -3
- package/dist/reset-images.d.ts.map +0 -1
- package/dist/reset-images.js.map +0 -1
- package/dist/reset-notes.d.ts +0 -3
- package/dist/reset-notes.d.ts.map +0 -1
- package/dist/reset-notes.js.map +0 -1
- package/dist/revert-articles.d.ts +0 -3
- package/dist/revert-articles.d.ts.map +0 -1
- package/dist/revert-articles.js.map +0 -1
- package/dist/revert-books.d.ts +0 -3
- package/dist/revert-books.d.ts.map +0 -1
- package/dist/revert-books.js.map +0 -1
- package/dist/revert-images.d.ts +0 -3
- package/dist/revert-images.d.ts.map +0 -1
- package/dist/revert-images.js.map +0 -1
- package/dist/revert-notes.d.ts +0 -3
- package/dist/revert-notes.d.ts.map +0 -1
- package/dist/revert-notes.js.map +0 -1
- package/dist/s-content-format-runner.d.ts +0 -10
- package/dist/s-content-format-runner.d.ts.map +0 -1
- package/dist/s-content-format-runner.js +0 -76
- package/dist/s-content-format-runner.js.map +0 -1
- package/dist/s-content-format.d.ts +0 -3
- package/dist/s-content-format.d.ts.map +0 -1
- package/dist/s-content-format.js.map +0 -1
- package/dist/s-content-format.test.d.ts +0 -2
- package/dist/s-content-format.test.d.ts.map +0 -1
- package/dist/s-content-format.test.js +0 -99
- package/dist/s-content-format.test.js.map +0 -1
- package/dist/update-json-articles.d.ts +0 -3
- package/dist/update-json-articles.d.ts.map +0 -1
- package/dist/update-json-articles.js.map +0 -1
- package/dist/update-raw-articles.d.ts +0 -3
- package/dist/update-raw-articles.d.ts.map +0 -1
- package/dist/update-raw-articles.js.map +0 -1
- package/dist/validate-json-articles.d.ts +0 -3
- package/dist/validate-json-articles.d.ts.map +0 -1
- package/dist/validate-json-articles.js.map +0 -1
package/dist/fetch-books.js
CHANGED
|
@@ -1,147 +1,166 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
createMinioClient
|
|
5
|
+
} from "./chunk-F4J6JBR4.js";
|
|
6
|
+
import {
|
|
7
|
+
createPrismaClient,
|
|
8
|
+
makeUnexportedStatus,
|
|
9
|
+
makeUserId
|
|
10
|
+
} from "./chunk-IWNZQXDG.js";
|
|
11
|
+
import {
|
|
12
|
+
createPushoverService
|
|
13
|
+
} from "./chunk-7PVU2Z32.js";
|
|
14
|
+
import "./chunk-JZOR5GA2.js";
|
|
15
|
+
|
|
16
|
+
// src/fetch-books.ts
|
|
6
17
|
import { dump } from "js-yaml";
|
|
7
18
|
import { access, mkdir, writeFile } from "node:fs/promises";
|
|
8
19
|
import path from "node:path";
|
|
9
20
|
import { setTimeout } from "node:timers/promises";
|
|
10
|
-
|
|
21
|
+
var OUTPUT_DIR = "markdown/book/";
|
|
11
22
|
function dumpFrontmatter(data) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
23
|
+
return dump(data, {
|
|
24
|
+
lineWidth: -1,
|
|
25
|
+
forceQuotes: false,
|
|
26
|
+
noRefs: true
|
|
27
|
+
});
|
|
17
28
|
}
|
|
18
29
|
async function withRetry(fn, retries = 3) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
console.warn(`⚠️ リトライ ${attempt + 1}/${retries} (${delay}ms後)...`);
|
|
28
|
-
await setTimeout(delay);
|
|
29
|
-
}
|
|
30
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
31
|
+
try {
|
|
32
|
+
return await fn();
|
|
33
|
+
} catch (error) {
|
|
34
|
+
if (attempt === retries - 1) throw error;
|
|
35
|
+
const delay = 1e3 * 2 ** attempt;
|
|
36
|
+
console.warn(`\u26A0\uFE0F \u30EA\u30C8\u30E9\u30A4 ${attempt + 1}/${retries} (${delay}ms\u5F8C)...`);
|
|
37
|
+
await setTimeout(delay);
|
|
30
38
|
}
|
|
31
|
-
|
|
39
|
+
}
|
|
40
|
+
throw new Error("unreachable");
|
|
32
41
|
}
|
|
33
42
|
async function exportData(data) {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
}
|
|
45
|
-
catch {
|
|
46
|
-
// ファイルが存在しない場合は書き出し
|
|
47
|
-
}
|
|
48
|
-
const frontmatter = dumpFrontmatter({
|
|
49
|
-
heading: item.isbn,
|
|
50
|
-
title: item.title,
|
|
51
|
-
draft: false,
|
|
52
|
-
rating: item.rating,
|
|
53
|
-
tags: item.tags,
|
|
54
|
-
googleSubtitle: item.googleSubTitle,
|
|
55
|
-
googleAuthors: item.googleAuthors,
|
|
56
|
-
googleDescription: item.googleDescription,
|
|
57
|
-
googleImgSrc: item.googleImgSrc,
|
|
58
|
-
googleHref: item.googleHref,
|
|
59
|
-
});
|
|
60
|
-
await writeFile(filePath, `---\n${frontmatter}---\n\n# ${item.title}\n`);
|
|
61
|
-
exportedCount++;
|
|
43
|
+
let skippedCount = 0;
|
|
44
|
+
let exportedCount = 0;
|
|
45
|
+
await mkdir(OUTPUT_DIR, { recursive: true });
|
|
46
|
+
for (const item of data) {
|
|
47
|
+
const filePath = `${OUTPUT_DIR}${item.isbn}.md`;
|
|
48
|
+
try {
|
|
49
|
+
await access(filePath);
|
|
50
|
+
skippedCount++;
|
|
51
|
+
continue;
|
|
52
|
+
} catch {
|
|
62
53
|
}
|
|
63
|
-
|
|
54
|
+
const frontmatter = dumpFrontmatter({
|
|
55
|
+
heading: item.isbn,
|
|
56
|
+
title: item.title,
|
|
57
|
+
draft: false,
|
|
58
|
+
rating: item.rating,
|
|
59
|
+
tags: item.tags,
|
|
60
|
+
googleSubtitle: item.googleSubTitle,
|
|
61
|
+
googleAuthors: item.googleAuthors,
|
|
62
|
+
googleDescription: item.googleDescription,
|
|
63
|
+
googleImgSrc: item.googleImgSrc,
|
|
64
|
+
googleHref: item.googleHref
|
|
65
|
+
});
|
|
66
|
+
await writeFile(filePath, `---
|
|
67
|
+
${frontmatter}---
|
|
68
|
+
|
|
69
|
+
# ${item.title}
|
|
70
|
+
`);
|
|
71
|
+
exportedCount++;
|
|
72
|
+
}
|
|
73
|
+
console.log(
|
|
74
|
+
`\u{1F4BE} Markdown: ${exportedCount} \u4EF6\u66F8\u304D\u51FA\u3057, ${skippedCount} \u4EF6\u30B9\u30AD\u30C3\u30D7`
|
|
75
|
+
);
|
|
64
76
|
}
|
|
65
77
|
async function main() {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
const env = {
|
|
79
|
+
DATABASE_URL: process.env.DATABASE_URL,
|
|
80
|
+
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
81
|
+
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
82
|
+
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
83
|
+
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
84
|
+
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME
|
|
85
|
+
};
|
|
86
|
+
if (Object.values(env).some((v) => !v)) {
|
|
87
|
+
throw new Error("Required environment variables are not set.");
|
|
88
|
+
}
|
|
89
|
+
if (process.env.MINIO_USE_SSL === "true" && (!process.env.CF_ACCESS_CLIENT_ID || !process.env.CF_ACCESS_CLIENT_SECRET)) {
|
|
90
|
+
throw new Error(
|
|
91
|
+
"CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true."
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
95
|
+
const notificationService = createPushoverService({
|
|
96
|
+
url: env.PUSHOVER_URL ?? "",
|
|
97
|
+
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
98
|
+
appToken: env.PUSHOVER_APP_TOKEN ?? ""
|
|
99
|
+
});
|
|
100
|
+
const minioClient = createMinioClient();
|
|
101
|
+
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
102
|
+
const UNEXPORTED = makeUnexportedStatus();
|
|
103
|
+
async function downloadBookImages(books) {
|
|
104
|
+
const outputDir = path.join(process.cwd(), "image/book");
|
|
105
|
+
await mkdir(outputDir, { recursive: true });
|
|
106
|
+
const booksWithImages = books.filter((book) => book.imagePath !== null);
|
|
107
|
+
console.log(
|
|
108
|
+
`\u{1F4F7} ${booksWithImages.length} \u4EF6\u306E\u66F8\u7C4D\u753B\u50CF\u3092\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9\u3057\u307E\u3059\u3002`
|
|
109
|
+
);
|
|
110
|
+
let skippedCount = 0;
|
|
111
|
+
let downloadedCount = 0;
|
|
112
|
+
for (const book of booksWithImages) {
|
|
113
|
+
const { isbn, imagePath } = book;
|
|
114
|
+
if (!imagePath) continue;
|
|
115
|
+
const ext = path.extname(imagePath);
|
|
116
|
+
const fileName = `${isbn}${ext}`;
|
|
117
|
+
const filePath = path.join(outputDir, fileName);
|
|
118
|
+
try {
|
|
119
|
+
await access(filePath);
|
|
120
|
+
skippedCount++;
|
|
121
|
+
continue;
|
|
122
|
+
} catch {
|
|
123
|
+
}
|
|
124
|
+
await withRetry(
|
|
125
|
+
() => minioClient.fGetObject(
|
|
126
|
+
env.MINIO_BUCKET_NAME ?? "",
|
|
127
|
+
`books/original/${imagePath}`,
|
|
128
|
+
filePath
|
|
129
|
+
)
|
|
130
|
+
);
|
|
131
|
+
downloadedCount++;
|
|
80
132
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
133
|
+
console.log(
|
|
134
|
+
`\u{1F4BE} \u66F8\u7C4D\u753B\u50CF: ${downloadedCount} \u4EF6\u30C0\u30A6\u30F3\u30ED\u30FC\u30C9, ${skippedCount} \u4EF6\u30B9\u30AD\u30C3\u30D7`
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
async function fetchBooks() {
|
|
138
|
+
const books = await prisma.book.findMany({
|
|
139
|
+
where: { userId, status: UNEXPORTED }
|
|
86
140
|
});
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
continue;
|
|
109
|
-
}
|
|
110
|
-
catch {
|
|
111
|
-
// ファイルが存在しない場合はダウンロード
|
|
112
|
-
}
|
|
113
|
-
await withRetry(() => minioClient.fGetObject(env.MINIO_BUCKET_NAME ?? "", `books/original/${imagePath}`, filePath));
|
|
114
|
-
downloadedCount++;
|
|
115
|
-
}
|
|
116
|
-
console.log(`💾 書籍画像: ${downloadedCount} 件ダウンロード, ${skippedCount} 件スキップ`);
|
|
117
|
-
}
|
|
118
|
-
async function fetchBooks() {
|
|
119
|
-
const books = await prisma.book.findMany({
|
|
120
|
-
where: { userId, status: UNEXPORTED },
|
|
121
|
-
});
|
|
122
|
-
console.log(`📊 ${books.length} 件のデータを取得しました。`);
|
|
123
|
-
await exportData(books);
|
|
124
|
-
await downloadBookImages(books);
|
|
125
|
-
}
|
|
126
|
-
try {
|
|
127
|
-
await fetchBooks();
|
|
128
|
-
await notificationService.notifyInfo("fetch-books completed", {
|
|
129
|
-
caller: "fetch-books",
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
catch (error) {
|
|
133
|
-
console.error("❌ エラーが発生しました:", error);
|
|
134
|
-
await notificationService.notifyError(`fetch-books failed: ${String(error)}`, {
|
|
135
|
-
caller: "fetch-books",
|
|
136
|
-
});
|
|
137
|
-
process.exit(1);
|
|
138
|
-
}
|
|
139
|
-
finally {
|
|
140
|
-
await prisma.$disconnect();
|
|
141
|
-
}
|
|
141
|
+
console.log(`\u{1F4CA} ${books.length} \u4EF6\u306E\u30C7\u30FC\u30BF\u3092\u53D6\u5F97\u3057\u307E\u3057\u305F\u3002`);
|
|
142
|
+
await exportData(books);
|
|
143
|
+
await downloadBookImages(books);
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
await fetchBooks();
|
|
147
|
+
await notificationService.notifyInfo("fetch-books completed", {
|
|
148
|
+
caller: "fetch-books"
|
|
149
|
+
});
|
|
150
|
+
} catch (error) {
|
|
151
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
152
|
+
await notificationService.notifyError(
|
|
153
|
+
`fetch-books failed: ${String(error)}`,
|
|
154
|
+
{
|
|
155
|
+
caller: "fetch-books"
|
|
156
|
+
}
|
|
157
|
+
);
|
|
158
|
+
process.exit(1);
|
|
159
|
+
} finally {
|
|
160
|
+
await prisma.$disconnect();
|
|
161
|
+
}
|
|
142
162
|
}
|
|
143
163
|
main().catch((error) => {
|
|
144
|
-
|
|
145
|
-
|
|
164
|
+
console.error(error);
|
|
165
|
+
process.exit(1);
|
|
146
166
|
});
|
|
147
|
-
//# sourceMappingURL=fetch-books.js.map
|
package/dist/fetch-images.js
CHANGED
|
@@ -1,85 +1,101 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
createMinioClient
|
|
5
|
+
} from "./chunk-F4J6JBR4.js";
|
|
6
|
+
import {
|
|
7
|
+
createPrismaClient,
|
|
8
|
+
makeUnexportedStatus,
|
|
9
|
+
makeUserId
|
|
10
|
+
} from "./chunk-IWNZQXDG.js";
|
|
11
|
+
import {
|
|
12
|
+
createPushoverService
|
|
13
|
+
} from "./chunk-7PVU2Z32.js";
|
|
14
|
+
import "./chunk-JZOR5GA2.js";
|
|
15
|
+
|
|
16
|
+
// src/fetch-images.ts
|
|
6
17
|
import { mkdir } from "node:fs/promises";
|
|
7
18
|
import path from "node:path";
|
|
8
19
|
import { setTimeout } from "node:timers/promises";
|
|
9
20
|
async function withRetry(fn, retries = 3) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
console.warn(`⚠️ リトライ ${attempt + 1}/${retries} (${delay}ms後)...`);
|
|
19
|
-
await setTimeout(delay);
|
|
20
|
-
}
|
|
21
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
22
|
+
try {
|
|
23
|
+
return await fn();
|
|
24
|
+
} catch (error) {
|
|
25
|
+
if (attempt === retries - 1) throw error;
|
|
26
|
+
const delay = 1e3 * 2 ** attempt;
|
|
27
|
+
console.warn(`\u26A0\uFE0F \u30EA\u30C8\u30E9\u30A4 ${attempt + 1}/${retries} (${delay}ms\u5F8C)...`);
|
|
28
|
+
await setTimeout(delay);
|
|
21
29
|
}
|
|
22
|
-
|
|
30
|
+
}
|
|
31
|
+
throw new Error("unreachable");
|
|
23
32
|
}
|
|
24
33
|
async function main() {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
34
|
+
const env = {
|
|
35
|
+
DATABASE_URL: process.env.DATABASE_URL,
|
|
36
|
+
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
37
|
+
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
38
|
+
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
39
|
+
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT,
|
|
40
|
+
MINIO_BUCKET_NAME: process.env.MINIO_BUCKET_NAME
|
|
41
|
+
};
|
|
42
|
+
if (Object.values(env).some((v) => !v)) {
|
|
43
|
+
throw new Error("Required environment variables are not set.");
|
|
44
|
+
}
|
|
45
|
+
if (process.env.MINIO_USE_SSL === "true" && (!process.env.CF_ACCESS_CLIENT_ID || !process.env.CF_ACCESS_CLIENT_SECRET)) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
"CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET are required when MINIO_USE_SSL is true."
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
51
|
+
const notificationService = createPushoverService({
|
|
52
|
+
url: env.PUSHOVER_URL ?? "",
|
|
53
|
+
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
54
|
+
appToken: env.PUSHOVER_APP_TOKEN ?? ""
|
|
55
|
+
});
|
|
56
|
+
const minioClient = createMinioClient();
|
|
57
|
+
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
58
|
+
const UNEXPORTED = makeUnexportedStatus();
|
|
59
|
+
async function fetchImages() {
|
|
60
|
+
const images = await prisma.image.findMany({
|
|
61
|
+
where: { userId, status: UNEXPORTED }
|
|
45
62
|
});
|
|
46
|
-
|
|
47
|
-
const
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
await withRetry(() => minioClient.fGetObject(env.MINIO_BUCKET_NAME ?? "", `images/original/${imagePath}`, filePath));
|
|
61
|
-
}
|
|
62
|
-
console.log("💾 すべての画像の保存が完了しました。");
|
|
63
|
-
}
|
|
64
|
-
try {
|
|
65
|
-
await fetchImages();
|
|
66
|
-
await notificationService.notifyInfo("fetch-images completed", {
|
|
67
|
-
caller: "fetch-images",
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
catch (error) {
|
|
71
|
-
console.error("❌ エラーが発生しました:", error);
|
|
72
|
-
await notificationService.notifyError(`fetch-images failed: ${String(error)}`, {
|
|
73
|
-
caller: "fetch-images",
|
|
74
|
-
});
|
|
75
|
-
process.exit(1);
|
|
76
|
-
}
|
|
77
|
-
finally {
|
|
78
|
-
await prisma.$disconnect();
|
|
63
|
+
console.log(`\u{1F4CA} \u53D6\u5F97\u3057\u305F\u753B\u50CF\u30C7\u30FC\u30BF\u6570: ${images.length}\u3002`);
|
|
64
|
+
const outputDir = path.join(process.cwd(), "image/dump");
|
|
65
|
+
await mkdir(outputDir, { recursive: true });
|
|
66
|
+
for (const image of images) {
|
|
67
|
+
const { path: imagePath } = image;
|
|
68
|
+
const filePath = path.join(outputDir, imagePath);
|
|
69
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
70
|
+
await withRetry(
|
|
71
|
+
() => minioClient.fGetObject(
|
|
72
|
+
env.MINIO_BUCKET_NAME ?? "",
|
|
73
|
+
`images/original/${imagePath}`,
|
|
74
|
+
filePath
|
|
75
|
+
)
|
|
76
|
+
);
|
|
79
77
|
}
|
|
78
|
+
console.log("\u{1F4BE} \u3059\u3079\u3066\u306E\u753B\u50CF\u306E\u4FDD\u5B58\u304C\u5B8C\u4E86\u3057\u307E\u3057\u305F\u3002");
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
await fetchImages();
|
|
82
|
+
await notificationService.notifyInfo("fetch-images completed", {
|
|
83
|
+
caller: "fetch-images"
|
|
84
|
+
});
|
|
85
|
+
} catch (error) {
|
|
86
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
87
|
+
await notificationService.notifyError(
|
|
88
|
+
`fetch-images failed: ${String(error)}`,
|
|
89
|
+
{
|
|
90
|
+
caller: "fetch-images"
|
|
91
|
+
}
|
|
92
|
+
);
|
|
93
|
+
process.exit(1);
|
|
94
|
+
} finally {
|
|
95
|
+
await prisma.$disconnect();
|
|
96
|
+
}
|
|
80
97
|
}
|
|
81
98
|
main().catch((error) => {
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
console.error(error);
|
|
100
|
+
process.exit(1);
|
|
84
101
|
});
|
|
85
|
-
//# sourceMappingURL=fetch-images.js.map
|
package/dist/fetch-notes.js
CHANGED
|
@@ -1,64 +1,76 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
2
|
+
import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
|
|
3
|
+
import {
|
|
4
|
+
createPrismaClient,
|
|
5
|
+
makeUnexportedStatus,
|
|
6
|
+
makeUserId
|
|
7
|
+
} from "./chunk-IWNZQXDG.js";
|
|
8
|
+
import {
|
|
9
|
+
createPushoverService
|
|
10
|
+
} from "./chunk-7PVU2Z32.js";
|
|
11
|
+
import "./chunk-JZOR5GA2.js";
|
|
12
|
+
|
|
13
|
+
// src/fetch-notes.ts
|
|
5
14
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
6
15
|
import { dirname } from "node:path";
|
|
7
|
-
|
|
16
|
+
var OUTPUT_DIR = "markdown/note/";
|
|
8
17
|
async function exportData(data) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
18
|
+
for (const item of data) {
|
|
19
|
+
const filePath = `${OUTPUT_DIR}${item.title}.md`;
|
|
20
|
+
await mkdir(dirname(filePath), { recursive: true });
|
|
21
|
+
await writeFile(filePath, `# ${item.title}
|
|
22
|
+
|
|
23
|
+
${item.markdown}
|
|
24
|
+
`);
|
|
25
|
+
}
|
|
14
26
|
}
|
|
15
27
|
async function main() {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
const env = {
|
|
29
|
+
DATABASE_URL: process.env.DATABASE_URL,
|
|
30
|
+
PUSHOVER_URL: process.env.PUSHOVER_URL,
|
|
31
|
+
PUSHOVER_USER_KEY: process.env.PUSHOVER_USER_KEY,
|
|
32
|
+
PUSHOVER_APP_TOKEN: process.env.PUSHOVER_APP_TOKEN,
|
|
33
|
+
USERNAME_TO_EXPORT: process.env.USERNAME_TO_EXPORT
|
|
34
|
+
};
|
|
35
|
+
if (Object.values(env).some((v) => !v)) {
|
|
36
|
+
throw new Error("Required environment variables are not set.");
|
|
37
|
+
}
|
|
38
|
+
const prisma = createPrismaClient(env.DATABASE_URL ?? "");
|
|
39
|
+
const notificationService = createPushoverService({
|
|
40
|
+
url: env.PUSHOVER_URL ?? "",
|
|
41
|
+
userKey: env.PUSHOVER_USER_KEY ?? "",
|
|
42
|
+
appToken: env.PUSHOVER_APP_TOKEN ?? ""
|
|
43
|
+
});
|
|
44
|
+
const userId = makeUserId(env.USERNAME_TO_EXPORT ?? "");
|
|
45
|
+
const UNEXPORTED = makeUnexportedStatus();
|
|
46
|
+
async function fetchNotes() {
|
|
47
|
+
const notes = await prisma.note.findMany({
|
|
48
|
+
where: { userId, status: UNEXPORTED },
|
|
49
|
+
select: { id: true, title: true, markdown: true }
|
|
31
50
|
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
});
|
|
54
|
-
process.exit(1);
|
|
55
|
-
}
|
|
56
|
-
finally {
|
|
57
|
-
await prisma.$disconnect();
|
|
58
|
-
}
|
|
51
|
+
console.log(`\u{1F4CA} ${notes.length} \u4EF6\u306E\u30C7\u30FC\u30BF\u3092\u53D6\u5F97\u3057\u307E\u3057\u305F\u3002`);
|
|
52
|
+
await exportData(notes);
|
|
53
|
+
console.log("\u{1F4BE} \u30C7\u30FC\u30BF\u304CMarkdown\u30D5\u30A1\u30A4\u30EB\u306B\u66F8\u304D\u51FA\u3055\u308C\u307E\u3057\u305F\u3002");
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
await fetchNotes();
|
|
57
|
+
await notificationService.notifyInfo("fetch-notes completed", {
|
|
58
|
+
caller: "fetch-notes"
|
|
59
|
+
});
|
|
60
|
+
} catch (error) {
|
|
61
|
+
console.error("\u274C \u30A8\u30E9\u30FC\u304C\u767A\u751F\u3057\u307E\u3057\u305F:", error);
|
|
62
|
+
await notificationService.notifyError(
|
|
63
|
+
`fetch-notes failed: ${String(error)}`,
|
|
64
|
+
{
|
|
65
|
+
caller: "fetch-notes"
|
|
66
|
+
}
|
|
67
|
+
);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
} finally {
|
|
70
|
+
await prisma.$disconnect();
|
|
71
|
+
}
|
|
59
72
|
}
|
|
60
73
|
main().catch((error) => {
|
|
61
|
-
|
|
62
|
-
|
|
74
|
+
console.error(error);
|
|
75
|
+
process.exit(1);
|
|
63
76
|
});
|
|
64
|
-
//# sourceMappingURL=fetch-notes.js.map
|