@s-hirano-ist/s-scripts 4.2.0 → 4.4.0

package/dist/chunk-KKWRXAAY.js

@@ -0,0 +1,447 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+
+// ../infrastructure/dist/observability/diagnostic-error.js
+function createDiagnosticError(env) {
+  const secrets = [
+    env.DATABASE_URL,
+    env.AUTH_SECRET,
+    env.AUTH0_CLIENT_SECRET,
+    env.LOCAL_AUTH_PASSWORD,
+    env.QDRANT_API_KEY,
+    env.S3_ACCESS_KEY_ID,
+    env.S3_SECRET_ACCESS_KEY,
+    env.WEB_PUSH_PRIVATE_KEY,
+    env.PUSHOVER_APP_TOKEN,
+    env.PUSHOVER_USER_KEY,
+    env.CRON_SECRET,
+    env.GOOGLE_BOOKS_API_KEY,
+    env.NEXT_PUBLIC_SENTRY_DSN
+  ].filter((value) => typeof value === "string" && value.length > 0);
+  function redact(input) {
+    let value = input;
+    for (const secret of secrets)
+      value = value.replaceAll(secret, "[REDACTED]");
+    return value.replaceAll(/\bBearer\s+[^\s,;]+/giu, "Bearer [REDACTED]").replaceAll(/\b(?:cookie|authorization|set-cookie)\s*[:=][^\r\n]*/giu, "[REDACTED HEADER]").replaceAll(/\b(?:https?|postgres(?:ql)?):\/\/\S+/giu, (match) => {
+      try {
+        const url = new URL(match);
+        if (!url.username && !url.password && !url.search && !url.hash)
+          return match;
+        if (url.username || url.password) {
+          url.username = "REDACTED";
+          url.password = "";
+        }
+        if (url.search || url.hash)
+          url.search = "?[REDACTED]";
+        url.hash = "";
+        return url.href;
+      } catch {
+        return "[REDACTED URL]";
+      }
+    });
+  }
+  return (error) => {
+    const diagnostic = new Error(error instanceof Error ? redact(error.message) : "Unknown operational error");
+    if (error instanceof Error) {
+      diagnostic.name = redact(error.name);
+      diagnostic.stack = error.stack ? redact(error.stack) : void 0;
+    }
+    return diagnostic;
+  };
+}
+
+// ../infrastructure/dist/observability/operational-telemetry.js
+function observe(action) {
+  try {
+    action();
+  } catch {
+  }
+}
+var jobs = /* @__PURE__ */ new Set([
+  "cleanup-minio-images",
+  "enrich-books",
+  "find-duplicate-json-articles",
+  "fix-book-frontmatter",
+  "fix-image-formats",
+  "validate-json-articles",
+  "update-json-articles",
+  "update-raw-articles",
+  "s-content-format",
+  "rag-delete-collection",
+  "rag-search",
+  "rag-ingest",
+  ...["fetch", "ingest", "reset", "revert"].flatMap((operation) => ["articles", "books", "images", "notes"].map((domain) => `${operation}-${domain}`))
+]);
+function telemetryJob(value) {
+  return jobs.has(value) ? value : "other";
+}
+function operationalAttributes(attributes = {}) {
+  return Object.fromEntries(Object.entries(attributes).filter((entry) => {
+    const [key, value] = entry;
+    return [
+      "tool",
+      "client",
+      "method",
+      "route",
+      "domain",
+      "status",
+      "job",
+      "sentry.environment",
+      "sentry.release",
+      "sentry.sdk.name",
+      "sentry.sdk.version",
+      "sentry.trace.parent_span_id",
+      "sentry.replay_id"
+    ].includes(key) && (typeof value === "string" || typeof value === "number");
+  }));
+}
+function createOperationalTelemetry(sdk, enabled) {
+  return {
+    log(level, message, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.logger[level](message, operationalAttributes(attributes)));
+    },
+    count(name, value, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.metrics.count(name, value, {
+          attributes: operationalAttributes(attributes)
+        }));
+    },
+    duration(name, milliseconds, attributes = {}) {
+      if (enabled())
+        observe(() => sdk.metrics.distribution(name, milliseconds, {
+          attributes: operationalAttributes(attributes),
+          unit: "millisecond"
+        }));
+    }
+  };
+}
+
+// ../infrastructure/dist/observability/log-level.js
+function mapStatusToLogLevel(status) {
+  if (status >= 500)
+    return "error";
+  if (status >= 400)
+    return "warn";
+  return "info";
+}
+
+// ../infrastructure/dist/observability/server-logger.js
+import pino from "pino";
+var pinoConfig = {
+  browser: { asObject: true }
+};
+var ServerLogger = class {
+  telemetry;
+  logger;
+  notificationService;
+  reporter;
+  constructor(notificationService, reporter, telemetry2) {
+    this.telemetry = telemetry2;
+    this.reporter = reporter;
+    this.logger = pino(pinoConfig);
+    this.notificationService = notificationService;
+  }
+  async info(message, context, options) {
+    await this.logWithLevel("info", message, context, void 0, options);
+  }
+  async warn(message, context, options) {
+    await this.logWithLevel("warn", message, context, void 0, options);
+  }
+  async error(message, context, error, options) {
+    await this.logWithLevel("error", message, context, error, options);
+  }
+  async logWithLevel(level, message, context, error, options) {
+    const expectedLevel = mapStatusToLogLevel(context.status);
+    const finalLevel = level === expectedLevel ? level : expectedLevel;
+    this.telemetry?.log(finalLevel, "Application operation", {
+      status: context.status
+    });
+    const logData = {
+      caller: context.caller,
+      status: context.status,
+      userId: context.userId,
+      ...context.additionalContext
+    };
+    if (level === "error") {
+      try {
+        this.captureError(message, context, error);
+      } catch (reportError) {
+        this.logger.warn({ error: reportError }, "Failed to report error");
+      }
+    }
+    if (error !== void 0) {
+      console.error("Additional error details:", error);
+    }
+    switch (finalLevel) {
+      case "info":
+        this.logger.info(logData, message);
+        break;
+      case "warn":
+        this.logger.warn(logData, message);
+        break;
+      case "error":
+        this.logger.error(logData, message);
+        break;
+    }
+    if (level === "error" && options?.severity === "critical" || level === "info" && options?.notifyCompletion) {
+      const notificationContext = {
+        caller: context.caller,
+        userId: context.userId
+      };
+      try {
+        switch (level) {
+          case "info":
+            await this.notificationService.notifyInfo(message, notificationContext);
+            break;
+          case "error":
+            await this.notificationService.notifyError(message, notificationContext);
+            break;
+        }
+      } catch (notificationError) {
+        this.logger.warn({ error: notificationError }, "Failed to send notification");
+        try {
+          this.reporter.captureException(notificationError, {
+            tags: {
+              caller: context.caller,
+              error_type: "notification_delivery",
+              status: String(context.status)
+            },
+            user: context.userId ? { id: context.userId } : void 0
+          });
+        } catch (reportError) {
+          this.logger.warn({ error: reportError }, "Failed to report notification failure");
+        }
+      }
+    }
+  }
+  captureError(message, context, error) {
+    const capturedError = this.findError(error);
+    const options = {
+      extra: {
+        additionalContext: context.additionalContext,
+        originalError: capturedError ? void 0 : error
+      },
+      tags: {
+        caller: context.caller,
+        error_type: capturedError?.name ?? "message",
+        status: String(context.status)
+      },
+      user: context.userId ? { id: context.userId } : void 0
+    };
+    if (capturedError) {
+      this.reporter.captureException(capturedError, options);
+      return;
+    }
+    this.reporter.captureMessage(message, { ...options, level: "error" });
+  }
+  findError(value) {
+    if (value instanceof Error)
+      return value;
+    if (typeof value === "object" && value !== null && "cause" in value && value.cause instanceof Error) {
+      return value.cause;
+    }
+    return void 0;
+  }
+};
+
+// ../notification/dist/errors.js
+var NotificationError = class extends Error {
+  constructor(message = "notificationSend") {
+    super(message);
+    this.name = "NotificationError";
+  }
+};
+
+// ../notification/dist/pushover.js
+var PushoverService = class {
+  config;
+  constructor(config) {
+    this.config = config;
+  }
+  /**
+   * Sends a message to the Pushover API.
+   *
+   * @param message - The message to send
+   * @param priority - The priority level (-1 = low, 0 = normal, 1 = high)
+   *
+   * @internal
+   */
+  async sendMessage(message, priority = 0) {
+    const body = new URLSearchParams({
+      token: this.config.appToken,
+      user: this.config.userKey,
+      message,
+      priority: String(priority)
+    });
+    const result = await fetch(this.config.url, {
+      method: "POST",
+      headers: { "Content-Type": "application/x-www-form-urlencoded" },
+      body,
+      signal: AbortSignal.timeout(5e3)
+    });
+    if (!result.ok)
+      throw new NotificationError();
+  }
+  async notifyError(message, context) {
+    const enrichedMessage = `[ERROR] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 1);
+  }
+  async notifyWarning(message, context) {
+    const enrichedMessage = `[WARN] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, 0);
+  }
+  async notifyInfo(message, context) {
+    const enrichedMessage = `[INFO] ${context.caller}: ${message}`;
+    await this.sendMessage(enrichedMessage, -1);
+  }
+};
+function createPushoverService(config) {
+  return new PushoverService(config);
+}
+
+// ../notification/dist/web-push.js
+import { sendNotification, setVapidDetails } from "web-push";
+
+// src/lib/cli-observability.ts
+import * as Sentry from "@sentry/node";
+var dryRun = () => process.argv.includes("--dry-run");
+var initialized = false;
+var telemetry = createOperationalTelemetry(
+  Sentry,
+  () => !dryRun() && process.env.NODE_ENV === "production" && Boolean(process.env.NEXT_PUBLIC_SENTRY_DSN)
+);
+function initialize() {
+  if (initialized || dryRun()) return;
+  initialized = true;
+  if (!process.env.NEXT_PUBLIC_SENTRY_DSN)
+    console.warn(
+      "NEXT_PUBLIC_SENTRY_DSN is not set; errors will only be logged locally and critical alerts sent via Pushover."
+    );
+  try {
+    Sentry.init({
+      dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
+      enabled: process.env.NODE_ENV === "production" && Boolean(process.env.NEXT_PUBLIC_SENTRY_DSN),
+      environment: process.env.VERCEL_ENV ?? process.env.NODE_ENV,
+      release: process.env.SENTRY_RELEASE ?? process.env.VERCEL_GIT_COMMIT_SHA,
+      beforeSendLog(log) {
+        log.attributes = operationalAttributes(log.attributes);
+        return log;
+      },
+      beforeSendMetric(metric) {
+        metric.attributes = operationalAttributes(metric.attributes);
+        return metric;
+      },
+      // The runner owns reporting and flushing, including handled item failures.
+      defaultIntegrations: false,
+      dataCollection: {
+        userInfo: false,
+        cookies: false,
+        httpHeaders: false,
+        httpBodies: [],
+        urlQueryParams: false,
+        databaseQueryData: false,
+        stackFrameVariables: false
+      },
+      beforeSend(event) {
+        delete event.user;
+        delete event.request;
+        delete event.breadcrumbs;
+        return event;
+      }
+    });
+  } catch (error) {
+    console.error(
+      "Failed to initialize Sentry",
+      createDiagnosticError(process.env)(error)
+    );
+  }
+}
+function logger() {
+  initialize();
+  const diagnose = createDiagnosticError(process.env);
+  return new ServerLogger(
+    createPushoverService({
+      url: process.env.PUSHOVER_URL ?? "https://api.pushover.net/1/messages.json",
+      userKey: process.env.PUSHOVER_USER_KEY ?? "",
+      appToken: process.env.PUSHOVER_APP_TOKEN ?? ""
+    }),
+    {
+      captureException: (error, context) => Sentry.captureException(diagnose(error), context),
+      captureMessage: (message, context) => Sentry.captureMessage(message, context)
+    }
+  );
+}
+function reportCliError(error, caller) {
+  const diagnostic = createDiagnosticError(process.env)(error);
+  console.error(diagnostic);
+  if (dryRun()) return;
+  initialize();
+  telemetry.log("error", "CLI item failed", { job: telemetryJob(caller) });
+  try {
+    Sentry.captureException(diagnostic, {
+      tags: { caller, severity: "normal" }
+    });
+  } catch (reportError) {
+    console.error(
+      "Failed to report error",
+      createDiagnosticError(process.env)(reportError)
+    );
+  }
+}
+var cliNotifications = {
+  async notifyInfo(message, context) {
+    if (dryRun()) return;
+    await logger().info(
+      message,
+      { ...context, caller: context.caller ?? "cli", status: 200 },
+      { notifyCompletion: true }
+    );
+  }
+};
+async function runCli(caller, execute) {
+  const started = performance.now();
+  const job = telemetryJob(caller);
+  let status = "ok";
+  try {
+    initialize();
+    telemetry.log("info", "CLI started", { job });
+    await execute();
+    telemetry.log("info", "CLI completed", { job, status: "ok" });
+  } catch (error) {
+    status = "error";
+    telemetry.log("error", "CLI failed", { job, status: "error" });
+    process.exitCode = 1;
+    const diagnostic = createDiagnosticError(process.env)(error);
+    console.error(diagnostic);
+    if (!dryRun())
+      await logger().error(
+        `${caller} failed`,
+        { caller, status: 500 },
+        diagnostic,
+        { severity: "critical" }
+      );
+  } finally {
+    telemetry.count("cli.runs", 1, { job, status });
+    telemetry.duration("cli.duration", performance.now() - started, {
+      job,
+      status
+    });
+    if (initialized && !dryRun()) {
+      try {
+        if (!await Sentry.flush(2e3))
+          console.error("Sentry flush timed out");
+      } catch (error) {
+        console.error(
+          "Sentry flush failed",
+          createDiagnosticError(process.env)(error)
+        );
+      }
+    }
+  }
+}
+
+export {
+  reportCliError,
+  cliNotifications,
+  runCli
+};

package/dist/{chunk-M6LFKOUH.js → chunk-MVYAZJB2.js}

@@ -57,13 +57,13 @@ function createObjectStorageClient(config = loadStorageConfig()) {
     async getObject(bucket, key) {
       const result = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
       if (!(result.Body instanceof Readable))
-        throw new Error("Object storage returned no Node readable body");
+        throw new TypeError("Object storage returned no Node readable body");
       return result.Body;
     },
     async downloadObject(bucket, key, destination) {
       const result = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
       if (!(result.Body instanceof Readable))
-        throw new Error("Object storage returned no Node readable body");
+        throw new TypeError("Object storage returned no Node readable body");
       await pipeline(result.Body, createWriteStream(destination));
     },
     async statObject(bucket, key) {

package/dist/{chunk-R5EI7JXR.js → chunk-QHM2WC7S.js}

@@ -8,7 +8,7 @@ import {
   createObjectStorageClient,
   createStorageService,
   isObjectNotFound
-} from "./chunk-M6LFKOUH.js";
+} from "./chunk-MVYAZJB2.js";
 import {
   CreatedAt,
   ExportedStatus,
@@ -22,8 +22,8 @@ import {
   withTransactionRetry
 } from "./chunk-RLAYINCA.js";
 import {
-  createPushoverService
-} from "./chunk-7PVU2Z32.js";
+  cliNotifications
+} from "./chunk-KKWRXAAY.js";
 
 // ../core/shared-kernel/entities/file-entity.ts
 import { z } from "zod";
@@ -542,11 +542,7 @@ async function runContentImport(domain, source) {
     throw new Error("DATABASE_URL and USERNAME_TO_EXPORT are required.");
   const dryRun = process.argv.includes("--dry-run");
   const prisma = createPrismaClient(databaseUrl);
-  const notification = createPushoverService({
-    url: process.env.PUSHOVER_URL ?? "",
-    userKey: process.env.PUSHOVER_USER_KEY ?? "",
-    appToken: process.env.PUSHOVER_APP_TOKEN ?? ""
-  });
+  const notification = cliNotifications;
   try {
     const mediaDomain = domain === "images" || domain === "books";
     const bucket = process.env.S3_BUCKET_NAME;
@@ -580,14 +576,7 @@ async function runContentImport(domain, source) {
     if (!dryRun)
       await notification.notifyInfo(`ingest-${domain} completed`, {
         caller: `ingest-${domain}`
-      }).catch((error) => console.error("\u901A\u77E5\u5931\u6557:", error));
-  } catch (error) {
-    console.error(error);
-    process.exitCode = 1;
-    if (!dryRun)
-      await notification.notifyError(`ingest-${domain} failed`, { caller: `ingest-${domain}` }).catch(
-        (notifyError) => console.error("\u901A\u77E5\u5931\u6557:", notifyError)
-      );
+      });
   } finally {
     await prisma.$disconnect();
   }

package/dist/chunk-SFKQ2RAP.js

@@ -0,0 +1,87 @@
+import { createRequire as __cliCreateRequire } from "node:module"; const require = __cliCreateRequire(import.meta.url);
+import {
+  ensureCollection,
+  search
+} from "./chunk-J4VLH4VA.js";
+import "./chunk-JZOR5GA2.js";
+
+// src/rag/search-command.ts
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const options = {};
+  const positional = [];
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (arg === "--top-k" && i + 1 < args.length) {
+      options.topK = Number(args[++i]);
+    } else if (arg === "--type" && i + 1 < args.length) {
+      options.type = args[++i];
+    } else if (arg === "--heading" && i + 1 < args.length) {
+      options.heading = args[++i];
+    } else if (arg === "--content-type" && i + 1 < args.length) {
+      const value = args[++i];
+      const types = value.split(",");
+      options.contentType = types.length === 1 ? types[0] : types;
+    } else {
+      positional.push(arg);
+    }
+  }
+  const query = positional.join(" ");
+  if (!query) {
+    console.error(
+      "Usage: rag-search <query> [--top-k N] [--type markdown_note|bookmark_json] [--heading HEADING] [--content-type articles|books|notes]"
+    );
+    process.exitCode = 1;
+    return void 0;
+  }
+  return { query, options };
+}
+async function searchContent(query, options = {}) {
+  const results = await search(query, {
+    topK: options.topK,
+    filter: {
+      type: options.type,
+      top_heading: options.heading,
+      content_type: options.contentType
+    }
+  });
+  return { results, query, totalResults: results.length };
+}
+async function search2() {
+  const args = parseArgs();
+  if (!args) return;
+  const { query, options } = args;
+  console.log(`Searching for: "${query}"`);
+  if (options.topK) console.log(`  top-k: ${options.topK}`);
+  if (options.type) console.log(`  type: ${options.type}`);
+  if (options.heading) console.log(`  heading: ${options.heading}`);
+  if (options.contentType)
+    console.log(
+      `  content-type: ${Array.isArray(options.contentType) ? options.contentType.join(",") : options.contentType}`
+    );
+  console.log();
+  await ensureCollection();
+  const response = await searchContent(query, options);
+  console.log(`Found ${response.totalResults} results:
+`);
+  for (const [i, result] of response.results.entries()) {
+    console.log(`--- Result ${i + 1} (score: ${result.score.toFixed(4)}) ---`);
+    console.log(`Title: ${result.title}`);
+    if (result.url) console.log(`URL: ${result.url}`);
+    console.log(`Type: ${result.type}`);
+    console.log(`Content-Type: ${result.content_type}`);
+    console.log(`Heading: ${result.heading_path.join(" > ")}`);
+    console.log(`
+${result.text}
+`);
+  }
+}
+async function main() {
+  if (!process.env.QDRANT_URL) {
+    throw new Error("QDRANT_URL environment variable is required.");
+  }
+  await search2();
+}
+export {
+  main
+};