@s-hirano-ist/s-scripts 4.1.0 → 4.3.0

package/README.md

@@ -73,3 +73,11 @@ validate-json-articles
 ## License
 
 AGPL-3.0
+
+## Operational notifications
+
+Set `NEXT_PUBLIC_SENTRY_DSN` to record CLI errors in Sentry. If it is missing, the command warns and continues with local diagnostics. Set `PUSHOVER_USER_KEY` and `PUSHOVER_APP_TOKEN` for completion and critical alerts; `PUSHOVER_URL` defaults to `https://api.pushover.net/1/messages.json`.
+
+Unexpected item failures go to Sentry only. A failure that aborts the command, including initialization or cleanup, also sends a critical Pushover alert and sets a nonzero exit code. Existing completion notifications use Pushover only; auxiliary commands do not add success notifications. Expected usage and content validation failures keep their nonzero exit codes without critical alerts.
+
+Notification delivery errors go to Sentry without changing the command's result or sending another Pushover alert. Commands wait up to two seconds for Sentry to flush before exiting; Pushover requests have a five-second timeout. Commands running with `--dry-run` do not initialize Sentry or send external notifications.

package/dist/{chunk-7XX57D7G.js → chunk-3INTMHPH.js}

@@ -2,7 +2,7 @@ import { createRequire as __cliCreateRequire } from "node:module"; const require
 import {
   makeExportedAt,
   withTransactionRetry
-} from "./chunk-DIG75T3J.js";
+} from "./chunk-RLAYINCA.js";
 
 // ../application/dist/batch/notes-batch-service.js
 var NotesBatchApplicationService = class {

package/dist/chunk-4CXH2QIX.js

@@ -0,0 +1,445 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../infrastructure/dist/observability/diagnostic-error.js
+function createDiagnosticError(env) {
+  const secrets = [
+    env.DATABASE_URL,
+    env.AUTH_SECRET,
+    env.AUTH0_CLIENT_SECRET,
+    env.LOCAL_AUTH_PASSWORD,
+    env.QDRANT_API_KEY,
+    env.S3_ACCESS_KEY_ID,
+    env.S3_SECRET_ACCESS_KEY,
+    env.WEB_PUSH_PRIVATE_KEY,
+    env.PUSHOVER_APP_TOKEN,
+    env.PUSHOVER_USER_KEY,
+    env.CRON_SECRET,
+    env.GOOGLE_BOOKS_API_KEY,
+    env.NEXT_PUBLIC_SENTRY_DSN
+  ].filter((value) => typeof value === "string" && value.length > 0);
+  function redact(input) {
+    let value = input;
+    for (const secret of secrets)
+      value = value.replaceAll(secret, "[REDACTED]");
+    return value.replaceAll(/\bBearer\s+[^\s,;]+/giu, "Bearer [REDACTED]").replaceAll(/\b(?:cookie|authorization|set-cookie)\s*[:=][^\r\n]*/giu, "[REDACTED HEADER]").replaceAll(/\b(?:https?|postgres(?:ql)?):\/\/\S+/giu, (match) => {
+      try {
+        const url = new URL(match);
+        if (!url.username && !url.password && !url.search && !url.hash)
+          return match;
+        if (url.username || url.password) {
+          url.username = "REDACTED";
+          url.password = "";
+        }
+        if (url.search || url.hash)
+          url.search = "?[REDACTED]";
+        url.hash = "";
+        return url.href;
+      } catch {
+        return "[REDACTED URL]";
+      }
+    });
+  }
+  return (error) => {
+    const diagnostic = new Error(error instanceof Error ? redact(error.message) : "Unknown operational error");
+    if (error instanceof Error) {
+      diagnostic.name = redact(error.name);
+      diagnostic.stack = error.stack ? redact(error.stack) : void 0;
+    }
+    return diagnostic;
+  };
+}
+
+// ../infrastructure/dist/observability/operational-telemetry.js
+function observe(action) {
+  try {
+    action();
+  } catch {
+  }
+}
+var jobs = /* @__PURE__ */ new Set([
+  "cleanup-minio-images",
+  "enrich-books",
+  "find-duplicate-json-articles",
+  "fix-book-frontmatter",
+  "fix-image-formats",
+  "validate-json-articles",
+  "update-json-articles",
+  "update-raw-articles",
+  "s-content-format",
+  "rag-delete-collection",
+  "rag-search",
+  "rag-ingest",
+  ...["fetch", "ingest", "reset", "revert"].flatMap((operation) => ["articles", "books", "images", "notes"].map((domain) => `${operation}-${domain}`))
+]);
+function telemetryJob(value) {
+  return jobs.has(value) ? value : "other";
+}
+function operationalAttributes(attributes = {}) {
+  return Object.fromEntries(Object.entries(attributes).filter((entry) => {
+    const [key, value] = entry;
+    return [
+      "method",
+      "route",
+      "domain",
+      "status",
+      "job",
+      "sentry.environment",
+      "sentry.release",
+      "sentry.sdk.name",
+      "sentry.sdk.version",
+      "sentry.trace.parent_span_id",
+      "sentry.replay_id"
+    ].includes(key) && (typeof value === "string" || typeof value === "number");
+  }));
+}
+function createOperationalTelemetry(sdk, enabled) {
+  return {
+    log(level, message, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.logger[level](message, operationalAttributes(attributes)));
+    },
+    count(name, value, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.metrics.count(name, value, {
+          attributes: operationalAttributes(attributes)
+        }));
+    },
+    duration(name, milliseconds, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.metrics.distribution(name, milliseconds, {
+          attributes: operationalAttributes(attributes),
+          unit: "millisecond"
+        }));
+    }
+  };
+}
+
+// ../infrastructure/dist/observability/log-level.js
+function mapStatusToLogLevel(status) {
+  if (status >= 500)
+    return "error";
+  if (status >= 400)
+    return "warn";
+  return "info";
+}
+
+// ../infrastructure/dist/observability/server-logger.js
+import pino from "pino";
+var pinoConfig = {
+  browser: { asObject: true }
+};
+var ServerLogger = class {
+  telemetry;
+  logger;
+  notificationService;
+  reporter;
+  constructor(notificationService, reporter, telemetry2) {
+    this.telemetry = telemetry2;
+    this.reporter = reporter;
+    this.logger = pino(pinoConfig);
+    this.notificationService = notificationService;
+  }
+  async info(message, context, options) {
+    await this.logWithLevel("info", message, context, void 0, options);
+  }
+  async warn(message, context, options) {
+    await this.logWithLevel("warn", message, context, void 0, options);
+  }
+  async error(message, context, error, options) {
+    await this.logWithLevel("error", message, context, error, options);
+  }
+  async logWithLevel(level, message, context, error, options) {
+    const expectedLevel = mapStatusToLogLevel(context.status);
+    const finalLevel = level === expectedLevel ? level : expectedLevel;
+    this.telemetry?.log(finalLevel, "Application operation", {
+      status: context.status
+    });
+    const logData = {
+      caller: context.caller,
+      status: context.status,
+      userId: context.userId,
+      ...context.additionalContext
+    };
+    if (level === "error") {
+      try {
+        this.captureError(message, context, error);
+      } catch (reportError) {
+        this.logger.warn({ error: reportError }, "Failed to report error");
+      }
+    }
+    if (error !== void 0) {
+      console.error("Additional error details:", error);
+    }
+    switch (finalLevel) {
+      case "info":
+        this.logger.info(logData, message);
+        break;
+      case "warn":
+        this.logger.warn(logData, message);
+        break;
+      case "error":
+        this.logger.error(logData, message);
+        break;
+    }
+    if (level === "error" && options?.severity === "critical" || level === "info" && options?.notifyCompletion) {
+      const notificationContext = {
+        caller: context.caller,
+        userId: context.userId
+      };
+      try {
+        switch (level) {
+          case "info":
+            await this.notificationService.notifyInfo(message, notificationContext);
+            break;
+          case "error":
+            await this.notificationService.notifyError(message, notificationContext);
+            break;
+        }
+      } catch (notificationError) {
+        this.logger.warn({ error: notificationError }, "Failed to send notification");
+        try {
+          this.reporter.captureException(notificationError, {
+            tags: {
+              caller: context.caller,
+              error_type: "notification_delivery",
+              status: String(context.status)
+            },
+            user: context.userId ? { id: context.userId } : void 0
+          });
+        } catch (reportError) {
+          this.logger.warn({ error: reportError }, "Failed to report notification failure");
+        }
+      }
+    }
+  }
+  captureError(message, context, error) {
+    const capturedError = this.findError(error);
+    const options = {
+      extra: {
+        additionalContext: context.additionalContext,
+        originalError: capturedError ? void 0 : error
+      },
+      tags: {
+        caller: context.caller,
+        error_type: capturedError?.name ?? "message",
+        status: String(context.status)
+      },
+      user: context.userId ? { id: context.userId } : void 0
+    };
+    if (capturedError) {
+      this.reporter.captureException(capturedError, options);
+      return;
+    }
+    this.reporter.captureMessage(message, { ...options, level: "error" });
+  }
+  findError(value) {
+    if (value instanceof Error)
+      return value;
+    if (typeof value === "object" && value !== null && "cause" in value && value.cause instanceof Error) {
+      return value.cause;
+    }
+    return void 0;
+  }
+};
+
+// ../notification/dist/errors.js
+var NotificationError = class extends Error {
+  constructor(message = "notificationSend") {
+    super(message);
+    this.name = "NotificationError";
+  }
+};
+
+// ../notification/dist/pushover.js
+var PushoverService = class {
+  config;
+  constructor(config) {
+    this.config = config;
+  }
+  /**
+   * Sends a message to the Pushover API.
+   *
+   * @param message - The message to send
+   * @param priority - The priority level (-1 = low, 0 = normal, 1 = high)
+   *
+   * @internal
+   */
+  async sendMessage(message, priority = 0) {
+    const body = new URLSearchParams({
+      token: this.config.appToken,
+      user: this.config.userKey,
+      message,
+      priority: String(priority)
+    });
+    const result = await fetch(this.config.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/x-www-form-urlencoded" },
+      body,
+      signal: AbortSignal.timeout(5e3)
+    });
+    if (!result.ok)
+      throw new NotificationError();
+  }
+  async notifyError(message, context) {
+    const enrichedMessage = `[ERROR] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 1);
+  }
+  async notifyWarning(message, context) {
+    const enrichedMessage = `[WARN] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 0);
+  }
+  async notifyInfo(message, context) {
+    const enrichedMessage = `[INFO] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, -1);
+  }
+};
+function createPushoverService(config) {
+  return new PushoverService(config);
+}
+
+// ../notification/dist/web-push.js
+import { sendNotification, setVapidDetails } from "web-push";
+
+// src/lib/cli-observability.ts
+import * as Sentry from "@sentry/node";
+var dryRun = () => process.argv.includes("--dry-run");
+var initialized = false;
+var telemetry = createOperationalTelemetry(
+  Sentry,
+  () => !dryRun() && process.env.NODE_ENV === "production" && Boolean(process.env.NEXT_PUBLIC_SENTRY_DSN)
+);
+function initialize() {
+  if (initialized || dryRun()) return;
+  initialized = true;
+  if (!process.env.NEXT_PUBLIC_SENTRY_DSN)
+    console.warn(
+      "NEXT_PUBLIC_SENTRY_DSN is not set; errors will only be logged locally and critical alerts sent via Pushover."
+    );
+  try {
+    Sentry.init({
+      dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
+      enabled: process.env.NODE_ENV === "production" && Boolean(process.env.NEXT_PUBLIC_SENTRY_DSN),
+      environment: process.env.VERCEL_ENV ?? process.env.NODE_ENV,
+      release: process.env.SENTRY_RELEASE ?? process.env.VERCEL_GIT_COMMIT_SHA,
+      beforeSendLog(log) {
+        log.attributes = operationalAttributes(log.attributes);
+        return log;
+      },
+      beforeSendMetric(metric) {
+        metric.attributes = operationalAttributes(metric.attributes);
+        return metric;
+      },
+      // The runner owns reporting and flushing, including handled item failures.
+      defaultIntegrations: false,
+      dataCollection: {
+        userInfo: false,
+        cookies: false,
+        httpHeaders: false,
+        httpBodies: [],
+        urlQueryParams: false,
+        databaseQueryData: false,
+        stackFrameVariables: false
+      },
+      beforeSend(event) {
+        delete event.user;
+        delete event.request;
+        delete event.breadcrumbs;
+        return event;
+      }
+    });
+  } catch (error) {
+    console.error(
+      "Failed to initialize Sentry",
+      createDiagnosticError(process.env)(error)
+    );
+  }
+}
+function logger() {
+  initialize();
+  const diagnose = createDiagnosticError(process.env);
+  return new ServerLogger(
+    createPushoverService({
+      url: process.env.PUSHOVER_URL ?? "https://api.pushover.net/1/messages.json",
+      userKey: process.env.PUSHOVER_USER_KEY ?? "",
+      appToken: process.env.PUSHOVER_APP_TOKEN ?? ""
+    }),
+    {
+      captureException: (error, context) => Sentry.captureException(diagnose(error), context),
+      captureMessage: (message, context) => Sentry.captureMessage(message, context)
+    }
+  );
+}
+function reportCliError(error, caller) {
+  const diagnostic = createDiagnosticError(process.env)(error);
+  console.error(diagnostic);
+  if (dryRun()) return;
+  initialize();
+  telemetry.log("error", "CLI item failed", { job: telemetryJob(caller) });
+  try {
+    Sentry.captureException(diagnostic, {
+      tags: { caller, severity: "normal" }
+    });
+  } catch (reportError) {
+    console.error(
+      "Failed to report error",
+      createDiagnosticError(process.env)(reportError)
+    );
+  }
+}
+var cliNotifications = {
+  async notifyInfo(message, context) {
+    if (dryRun()) return;
+    await logger().info(
+      message,
+      { ...context, caller: context.caller ?? "cli", status: 200 },
+      { notifyCompletion: true }
+    );
+  }
+};
+async function runCli(caller, execute) {
+  const started = performance.now();
+  const job = telemetryJob(caller);
+  let status = "ok";
+  try {
+    initialize();
+    telemetry.log("info", "CLI started", { job });
+    await execute();
+    telemetry.log("info", "CLI completed", { job, status: "ok" });
+  } catch (error) {
+    status = "error";
+    telemetry.log("error", "CLI failed", { job, status: "error" });
+    process.exitCode = 1;
+    const diagnostic = createDiagnosticError(process.env)(error);
+    console.error(diagnostic);
+    if (!dryRun())
+      await logger().error(
+        `${caller} failed`,
+        { caller, status: 500 },
+        diagnostic,
+        { severity: "critical" }
+      );
+  } finally {
+    telemetry.count("cli.runs", 1, { job, status });
+    telemetry.duration("cli.duration", performance.now() - started, {
+      job,
+      status
+    });
+    if (initialized && !dryRun()) {
+      try {
+        if (!await Sentry.flush(2e3))
+          console.error("Sentry flush timed out");
+      } catch (error) {
+        console.error(
+          "Sentry flush failed",
+          createDiagnosticError(process.env)(error)
+        );
+      }
+    }
+  }
+}
+
+export {
+  reportCliError,
+  cliNotifications,
+  runCli
+};

package/dist/chunk-67LIMNXH.js

@@ -0,0 +1,36 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  deleteCollection,
+  getQdrantClient
+} from "./chunk-J4VLH4VA.js";
+import "./chunk-JZOR5GA2.js";
+
+// src/rag/delete-collection-command.ts
+async function main() {
+  const collectionName = process.argv[2];
+  if (!collectionName) {
+    console.error(
+      "Usage: rag-delete-collection <collection-name>\nExample: rag-delete-collection knowledge_v1"
+    );
+    process.exitCode = 1;
+    return;
+  }
+  if (!process.env.QDRANT_URL) {
+    throw new Error("QDRANT_URL environment variable is required.");
+  }
+  const qdrant = getQdrantClient();
+  const collections = await qdrant.getCollections();
+  const exists = collections.collections.some((c) => c.name === collectionName);
+  if (!exists) {
+    console.log(
+      `Collection "${collectionName}" does not exist. Nothing to delete.`
+    );
+    return;
+  }
+  console.log(`Deleting collection "${collectionName}"...`);
+  await deleteCollection(collectionName);
+  console.log(`Collection "${collectionName}" deleted successfully.`);
+}
+export {
+  main
+};

package/dist/{chunk-OCEV2QX3.js → chunk-BYEEIQ7O.js}

@@ -2,7 +2,7 @@ import { createRequire as __cliCreateRequire } from "node:module"; const require
 import {
   makeExportedAt,
   withTransactionRetry
-} from "./chunk-DIG75T3J.js";
+} from "./chunk-RLAYINCA.js";
 
 // ../application/dist/batch/articles-batch-service.js
 var ArticlesBatchApplicationService = class {

package/dist/{chunk-2PZXXOYG.js → chunk-GBQOCN2C.js}

@@ -4,9 +4,11 @@ import {
   readImageMetadata
 } from "./chunk-KKZP7ZF3.js";
 import {
-  createMinioClient,
-  createStorageService
-} from "./chunk-F4J6JBR4.js";
+  StorageOperationError,
+  createObjectStorageClient,
+  createStorageService,
+  isObjectNotFound
+} from "./chunk-MVYAZJB2.js";
 import {
   CreatedAt,
   ExportedStatus,
@@ -18,10 +20,10 @@ import {
   import_generated,
   makeId,
   withTransactionRetry
-} from "./chunk-DIG75T3J.js";
+} from "./chunk-RLAYINCA.js";
 import {
-  createPushoverService
-} from "./chunk-7PVU2Z32.js";
+  cliNotifications
+} from "./chunk-4CXH2QIX.js";
 
 // ../core/shared-kernel/entities/file-entity.ts
 import { z } from "zod";
@@ -497,7 +499,9 @@ function createImportMediaStore(storage, processor) {
       const target = storage[domain];
       try {
         await target.getImageOrThrow(path, false);
-      } catch {
+      } catch (error) {
+        if (!isObjectNotFound(error instanceof StorageOperationError ? error.cause : error))
+          throw error;
         await target.uploadImage(path, bytes, false);
       }
       await target.uploadImage(path, await processor.createThumbnail(bytes, 192, 192), true);
@@ -512,16 +516,16 @@ function createImportMediaStore(storage, processor) {
 // ../infrastructure/dist/shared/books-storage-service.js
 var ORIGINAL_BOOK_IMAGE_PATH = "books/original";
 var THUMBNAIL_BOOK_IMAGE_PATH = "books/thumbnail";
-function createBooksStorageService(minioClient, bucketName) {
-  return createStorageService(minioClient, bucketName, {
+function createBooksStorageService(storageClient, bucketName) {
+  return createStorageService(storageClient, bucketName, {
     originalPrefix: ORIGINAL_BOOK_IMAGE_PATH,
     thumbnailPrefix: THUMBNAIL_BOOK_IMAGE_PATH
   });
 }
 
-// ../infrastructure/dist/shared/minio-storage-service.js
-function createImagesStorageService(minioClient, bucketName) {
-  return createStorageService(minioClient, bucketName, {
+// ../infrastructure/dist/shared/images-storage-service.js
+function createImagesStorageService(storageClient, bucketName) {
+  return createStorageService(storageClient, bucketName, {
     originalPrefix: "images/original",
     thumbnailPrefix: "images/thumbnail"
   });
@@ -538,17 +542,13 @@ async function runContentImport(domain, source) {
     throw new Error("DATABASE_URL and USERNAME_TO_EXPORT are required.");
   const dryRun = process.argv.includes("--dry-run");
   const prisma = createPrismaClient(databaseUrl);
-  const notification = createPushoverService({
-    url: process.env.PUSHOVER_URL ?? "",
-    userKey: process.env.PUSHOVER_USER_KEY ?? "",
-    appToken: process.env.PUSHOVER_APP_TOKEN ?? ""
-  });
+  const notification = cliNotifications;
   try {
     const mediaDomain = domain === "images" || domain === "books";
-    const bucket = process.env.MINIO_BUCKET_NAME;
+    const bucket = process.env.S3_BUCKET_NAME;
     if (mediaDomain && !dryRun && !bucket)
-      throw new Error("MINIO_BUCKET_NAME is required.");
-    const client = mediaDomain && !dryRun ? createMinioClient() : void 0;
+      throw new Error("S3_BUCKET_NAME is required.");
+    const client = mediaDomain && !dryRun ? createObjectStorageClient() : void 0;
     const media = client && bucket ? createImportMediaStore(
       {
         images: createImagesStorageService(client, bucket),
@@ -573,11 +573,10 @@ async function runContentImport(domain, source) {
       }
     );
     console.log(summary);
-    await notification.notifyInfo(`ingest-${domain} completed`, { caller: `ingest-${domain}` }).catch((error) => console.error("\u901A\u77E5\u5931\u6557:", error));
-  } catch (error) {
-    console.error(error);
-    process.exitCode = 1;
-    await notification.notifyError(`ingest-${domain} failed`, { caller: `ingest-${domain}` }).catch((notifyError) => console.error("\u901A\u77E5\u5931\u6557:", notifyError));
+    if (!dryRun)
+      await notification.notifyInfo(`ingest-${domain} completed`, {
+        caller: `ingest-${domain}`
+      });
   } finally {
     await prisma.$disconnect();
   }