gcp-job-runner 1.9.0 → 1.10.0

package/README.md

@@ -44,7 +44,7 @@ Same code, same arguments, same secrets. The cloud command automatically builds
 - **Cloud Run deployment** — no Terraform or manual GCP config needed, just `job cloud run`
 - **Smart caching** — a single Docker image contains all jobs; running different jobs or different arguments doesn't rebuild
 - **GCP Secret Manager** — secrets are loaded transparently for both local and cloud execution
-- **Unified file I/O** — `getFileWriter()` writes JSON/CSV/binary to a configured output location and `getInputFilesPath()` exposes the input location for reads, targeting local directories during development and Cloud Storage buckets in production with the same handler code
+- **Unified file I/O** — `getFileWriter()` writes JSON/CSV/binary to a configured output location and `readInputJson()` / `readInputText()` / `readInputBuffer()` read from a configured input location, targeting local directories during development and Cloud Storage buckets in production with the same handler code
 - **Multi-environment** — configure staging, production, etc. and switch with a single argument
 
 ## Install

package/dist/files.d.mts

@@ -77,6 +77,34 @@ declare function fileInput(): z.ZodString;
  * Throws when no input destination is configured.
  */
 declare function listInputFiles(): Promise<string[]>;
+/**
+ * Read a file from the configured input files destination as raw bytes.
+ * Local paths and `gs://` URIs are handled transparently.
+ *
+ * `relativePath` is sanitized the same way the writer sanitizes output
+ * paths — absolute paths and upward traversal (`..`) are rejected so a
+ * job can't escape its configured destination.
+ *
+ * Throws when no input destination is configured.
+ */
+declare function readInputBuffer(relativePath: string): Promise<Buffer>;
+/**
+ * Read a UTF-8 text file from the configured input files destination.
+ * Use this for CSV, JSON, plain text, SVG — anything character-based.
+ *
+ * See `readInputBuffer()` for path-sanitization and configuration
+ * semantics.
+ */
+declare function readInputText(relativePath: string): Promise<string>;
+/**
+ * Read and `JSON.parse` a file from the configured input files
+ * destination. Parse errors from a malformed file propagate as the
+ * standard `SyntaxError` thrown by `JSON.parse`.
+ *
+ * See `readInputBuffer()` for path-sanitization and configuration
+ * semantics.
+ */
+declare function readInputJson<T = unknown>(relativePath: string): Promise<T>;
 //#endregion
-export { FileWriter, fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles };
+export { FileWriter, fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles, readInputBuffer, readInputJson, readInputText };
 //# sourceMappingURL=files.d.mts.map

package/dist/files.mjs

@@ -1,7 +1,7 @@
 import path from "node:path";
 import { consola } from "consola";
 import { z } from "zod";
-import { mkdir, readdir, writeFile } from "node:fs/promises";
+import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 
 //#region src/files.ts
 const GCS_PREFIX = "gs://";
@@ -77,6 +77,52 @@ async function listInputFiles() {
 	if (destination.startsWith(GCS_PREFIX)) return listGcsFiles(destination);
 	return listLocalFiles(path.resolve(destination));
 }
+/**
+* Read a file from the configured input files destination as raw bytes.
+* Local paths and `gs://` URIs are handled transparently.
+*
+* `relativePath` is sanitized the same way the writer sanitizes output
+* paths — absolute paths and upward traversal (`..`) are rejected so a
+* job can't escape its configured destination.
+*
+* Throws when no input destination is configured.
+*/
+async function readInputBuffer(relativePath) {
+	return readInputRaw(relativePath);
+}
+/**
+* Read a UTF-8 text file from the configured input files destination.
+* Use this for CSV, JSON, plain text, SVG — anything character-based.
+*
+* See `readInputBuffer()` for path-sanitization and configuration
+* semantics.
+*/
+async function readInputText(relativePath) {
+	return (await readInputRaw(relativePath)).toString("utf8");
+}
+/**
+* Read and `JSON.parse` a file from the configured input files
+* destination. Parse errors from a malformed file propagate as the
+* standard `SyntaxError` thrown by `JSON.parse`.
+*
+* See `readInputBuffer()` for path-sanitization and configuration
+* semantics.
+*/
+async function readInputJson(relativePath) {
+	const text = await readInputText(relativePath);
+	return JSON.parse(text);
+}
+async function readInputRaw(relativePath) {
+	const destination = readInputDestination();
+	const safeRelative = sanitizeRelativePath(relativePath);
+	if (destination.startsWith(GCS_PREFIX)) return readGcsObject(destination, safeRelative);
+	return readFile(path.join(path.resolve(destination), safeRelative));
+}
+async function readGcsObject(uri, safeRelative) {
+	const { bucket, prefix } = parseGcsUri(uri);
+	const [buf] = await (await getStorageClient()).bucket(bucket).file(joinGcsPath(prefix, safeRelative)).download();
+	return buf;
+}
 async function listLocalFiles(basePath) {
 	return (await readdir(basePath, { withFileTypes: true })).filter((entry) => entry.isFile()).map((entry) => entry.name).sort();
 }
@@ -231,5 +277,5 @@ function contentTypeFor(relativePath) {
 }
 
 //#endregion
-export { fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles };
+export { fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles, readInputBuffer, readInputJson, readInputText };
 //# sourceMappingURL=files.mjs.map

package/dist/files.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"files.mjs","names":[],"sources":["../src/files.ts"],"sourcesContent":["import { mkdir, readdir, writeFile } from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { consola } from \"consola\";\nimport { z } from \"zod\";\nimport type { Storage } from \"@google-cloud/storage\";\n\n/**\n * Writer for job output files, produced by `getFileWriter()`. A single\n * writer instance targets the destination configured via the\n * `outputFilesPath` runner option (local directory or `gs://` URI). All\n * methods return the resolved absolute path or `gs://` URI of the written\n * file.\n */\nexport interface FileWriter {\n  /**\n   * Write a value as pretty-printed JSON (2-space indent, trailing newline).\n   * The `.json` extension is added if missing.\n   */\n  writeJson(relativePath: string, data: unknown): Promise<string>;\n  /** Write a UTF-8 string (e.g. CSV, SVG, plain text). */\n  writeText(relativePath: string, content: string): Promise<string>;\n  /** Write a binary buffer. */\n  writeBuffer(\n    relativePath: string,\n    content: Buffer | Uint8Array,\n  ): Promise<string>;\n}\n\nconst GCS_PREFIX = \"gs://\";\n\n/**\n * Return the resolved input files destination — where the job reads from.\n * Local paths are resolved to an absolute filesystem path; `gs://` URIs\n * are validated and canonicalized (trailing slashes stripped). Use this\n * to locate fixtures, reference datasets, or files produced by another\n * job, and read them with `node:fs` or `@google-cloud/storage` directly.\n *\n * Throws when no input destination is configured or the configured\n * `gs://` URI is malformed (e.g. missing bucket).\n */\nexport function getInputFilesPath(): string {\n  return resolveDestination(readInputDestination());\n}\n\n/**\n * Return the resolved output files destination — where `getFileWriter()`\n * writes to. Local paths are resolved to an absolute filesystem path;\n * `gs://` URIs are validated and canonicalized (trailing slashes\n * stripped). Useful when a job needs to read back its own artifacts\n * (e.g., to chain steps within a handler) or pass the destination to\n * another tool.\n *\n * Throws when no output destination is configured or the configured\n * `gs://` URI is malformed (e.g. missing bucket).\n */\nexport function getOutputFilesPath(): string {\n  return resolveDestination(readOutputDestination());\n}\n\n/**\n * Return a writer that persists files to the destination configured via\n * the `outputFilesPath` runner option. Local paths are used for local\n * execution; `gs://bucket[/prefix]` URIs are used for Cloud Run\n * deployments.\n *\n * Throws when no output destination is configured.\n */\nexport function getFileWriter(): FileWriter {\n  const destination = readOutputDestination();\n\n  if (destination.startsWith(GCS_PREFIX)) {\n    return createGcsWriter(destination);\n  }\n\n  return createLocalWriter(destination);\n}\n\n/**\n * Mark a Zod string field as a file input. Interactive mode detects this\n * marker and offers a selectable list of files from `getInputFilesPath()`\n * instead of a free-text prompt.\n *\n * ```ts\n * z.object({\n *   file: fileInput().describe(\"CSV to process\"),\n * });\n * ```\n *\n * Chains with `.describe()`, `.optional()`, `.default()` as usual.\n */\nexport function fileInput() {\n  return z.string().meta({ kind: \"file\" });\n}\n\n/**\n * List filenames available under the configured input files destination.\n * Returns names relative to the destination directory (no leading slash).\n * Results are sorted alphabetically.\n *\n * Local destinations list only top-level files (nested directories are\n * skipped for now). `gs://` destinations list every object under the\n * configured prefix.\n *\n * Throws when no input destination is configured.\n */\nexport async function listInputFiles(): Promise<string[]> {\n  const destination = readInputDestination();\n  if (destination.startsWith(GCS_PREFIX)) {\n    return listGcsFiles(destination);\n  }\n  return listLocalFiles(path.resolve(destination));\n}\n\nasync function listLocalFiles(basePath: string): Promise<string[]> {\n  const entries = await readdir(basePath, { withFileTypes: true });\n  return entries\n    .filter((entry) => entry.isFile())\n    .map((entry) => entry.name)\n    .sort();\n}\n\nasync function listGcsFiles(uri: string): Promise<string[]> {\n  const { bucket, prefix } = parseGcsUri(uri);\n\n  /**\n   * Terminate the list prefix with `/` so GCS treats it as a folder and\n   * not a bare substring — otherwise a configured `inputs` would also\n   * match sibling keys like `inputs2/…` or `inputs-backup/…`.\n   */\n  const listPrefix = prefix ? `${prefix}/` : undefined;\n  const stripLen = listPrefix ? listPrefix.length : 0;\n\n  const storage = await getStorageClient();\n  const [files] = await storage.bucket(bucket).getFiles({\n    prefix: listPrefix,\n  });\n\n  return files\n    .map((file) => file.name.slice(stripLen))\n    .filter((name) => name.length > 0 && !name.endsWith(\"/\"))\n    .sort();\n}\n\nfunction resolveDestination(destination: string): string {\n  if (destination.startsWith(GCS_PREFIX)) {\n    /**\n     * Validate and canonicalize via the same parser the writer uses, so\n     * read and write paths stay consistent — reject `gs://` without a\n     * bucket and strip any trailing slash from the prefix.\n     */\n    const { bucket, prefix } = parseGcsUri(destination);\n    return prefix\n      ? `${GCS_PREFIX}${bucket}/${prefix}`\n      : `${GCS_PREFIX}${bucket}`;\n  }\n  return path.resolve(destination);\n}\n\nfunction readInputDestination(): string {\n  const destination = process.env.JOB_INPUT_FILES_PATH;\n\n  if (!destination) {\n    throw new Error(\n      \"No input files destination configured.\\n\" +\n        \"Set `localInputFilesPath` or the current environment's \" +\n        \"`inputFilesPath` in your job-runner.config.ts, or set \" +\n        \"JOB_INPUT_FILES_PATH directly in the environment.\",\n    );\n  }\n\n  return destination;\n}\n\nfunction readOutputDestination(): string {\n  const destination = process.env.JOB_OUTPUT_FILES_PATH;\n\n  if (!destination) {\n    throw new Error(\n      \"No output files destination configured.\\n\" +\n        \"Set `localOutputFilesPath` or the current environment's \" +\n        \"`outputFilesPath` in your job-runner.config.ts, or set \" +\n        \"JOB_OUTPUT_FILES_PATH directly in the environment.\",\n    );\n  }\n\n  return destination;\n}\n\nfunction createLocalWriter(basePath: string): FileWriter {\n  const resolvedBase = path.resolve(basePath);\n\n  /** Takes an already-sanitized relative path and writes the content. */\n  async function write(\n    safeRelative: string,\n    content: string | Uint8Array,\n  ): Promise<string> {\n    const fullPath = path.join(resolvedBase, safeRelative);\n\n    await mkdir(path.dirname(fullPath), { recursive: true });\n    await writeFile(fullPath, content);\n\n    consola.info(`File written: ${fullPath}`);\n    return fullPath;\n  }\n\n  return {\n    async writeJson(relativePath, data) {\n      /** Sanitize before appending the extension so \"\" doesn't become \".json\". */\n      const withExtension = ensureExtension(\n        sanitizeRelativePath(relativePath),\n        \".json\",\n      );\n      return write(withExtension, formatJson(data));\n    },\n    async writeText(relativePath, content) {\n      return write(sanitizeRelativePath(relativePath), content);\n    },\n    async writeBuffer(relativePath, content) {\n      return write(sanitizeRelativePath(relativePath), content);\n    },\n  };\n}\n\ninterface GcsTarget {\n  bucket: string;\n  prefix: string;\n}\n\nfunction createGcsWriter(uri: string): FileWriter {\n  const target = parseGcsUri(uri);\n\n  /** Takes an already-sanitized relative path and uploads the content. */\n  async function write(\n    safeRelative: string,\n    content: string | Buffer | Uint8Array,\n    contentType: string,\n  ): Promise<string> {\n    const objectName = joinGcsPath(target.prefix, safeRelative);\n    const fullUri = `${GCS_PREFIX}${target.bucket}/${objectName}`;\n\n    const storage = await getStorageClient();\n    const file = storage.bucket(target.bucket).file(objectName);\n\n    const body =\n      typeof content === \"string\" || Buffer.isBuffer(content)\n        ? content\n        : Buffer.from(content);\n\n    await file.save(body, {\n      contentType,\n      resumable: false,\n    });\n\n    consola.info(`File written: ${fullUri}`);\n    return fullUri;\n  }\n\n  return {\n    async writeJson(relativePath, data) {\n      /** Sanitize before appending the extension so \"\" doesn't become \".json\". */\n      const withExtension = ensureExtension(\n        sanitizeRelativePath(relativePath),\n        \".json\",\n      );\n      return write(withExtension, formatJson(data), \"application/json\");\n    },\n    async writeText(relativePath, content) {\n      const safe = sanitizeRelativePath(relativePath);\n      return write(safe, content, contentTypeFor(safe));\n    },\n    async writeBuffer(relativePath, content) {\n      return write(\n        sanitizeRelativePath(relativePath),\n        content,\n        \"application/octet-stream\",\n      );\n    },\n  };\n}\n\n/**\n * Lazy Storage client singleton. The `@google-cloud/storage` module is only\n * loaded when a `gs://` destination is actually used for a write.\n */\nlet storageClient: Storage | null = null;\nasync function getStorageClient(): Promise<Storage> {\n  if (storageClient) return storageClient;\n  const { Storage: StorageCtor } = await import(\"@google-cloud/storage\");\n  storageClient = new StorageCtor();\n  return storageClient;\n}\n\nfunction parseGcsUri(uri: string): GcsTarget {\n  const withoutScheme = uri.slice(GCS_PREFIX.length);\n  const slashIndex = withoutScheme.indexOf(\"/\");\n\n  if (slashIndex === -1) {\n    if (!withoutScheme) {\n      throw new Error(`Invalid GCS URI: \"${uri}\" (missing bucket name)`);\n    }\n    return { bucket: withoutScheme, prefix: \"\" };\n  }\n\n  const bucket = withoutScheme.slice(0, slashIndex);\n  const prefix = withoutScheme.slice(slashIndex + 1).replace(/\\/+$/, \"\");\n\n  if (!bucket) {\n    throw new Error(`Invalid GCS URI: \"${uri}\" (missing bucket name)`);\n  }\n\n  return { bucket, prefix };\n}\n\nfunction joinGcsPath(prefix: string, relative: string): string {\n  const normalizedRelative = relative.replace(/^\\/+/, \"\");\n  return prefix ? `${prefix}/${normalizedRelative}` : normalizedRelative;\n}\n\nfunction sanitizeRelativePath(relativePath: string): string {\n  if (!relativePath || relativePath.trim() === \"\") {\n    throw new Error(\"File path is empty\");\n  }\n\n  if (path.isAbsolute(relativePath) || relativePath.startsWith(\"/\")) {\n    throw new Error(\n      `File path must be relative, got \"${relativePath}\". ` +\n        \"Absolute paths are not allowed.\",\n    );\n  }\n\n  const normalized = path.posix.normalize(relativePath.replace(/\\\\/g, \"/\"));\n\n  /**\n   * Reject `..` only as a distinct path segment, not as a prefix. A filename\n   * like \"..hidden\" is a legitimate dotfile variant, while \"..\" / \"../x\" /\n   * \"a/../../x\" all produce a `..` segment after normalization.\n   */\n  if (normalized.split(\"/\").includes(\"..\")) {\n    throw new Error(\n      `File path must not traverse upward, got \"${relativePath}\".`,\n    );\n  }\n\n  return normalized;\n}\n\nfunction ensureExtension(relativePath: string, extension: string): string {\n  return relativePath.endsWith(extension)\n    ? relativePath\n    : `${relativePath}${extension}`;\n}\n\nfunction formatJson(data: unknown): string {\n  return `${JSON.stringify(data, null, 2)}\\n`;\n}\n\nconst TEXT_CONTENT_TYPES: Record<string, string> = {\n  \".csv\": \"text/csv; charset=utf-8\",\n  \".json\": \"application/json\",\n  \".svg\": \"image/svg+xml\",\n  \".txt\": \"text/plain; charset=utf-8\",\n  \".html\": \"text/html; charset=utf-8\",\n  \".xml\": \"application/xml\",\n  \".yaml\": \"application/yaml\",\n  \".yml\": \"application/yaml\",\n  \".md\": \"text/markdown; charset=utf-8\",\n};\n\nfunction contentTypeFor(relativePath: string): string {\n  const ext = path.posix.extname(relativePath).toLowerCase();\n  return TEXT_CONTENT_TYPES[ext] ?? \"text/plain; charset=utf-8\";\n}\n"],"mappings":";;;;;;AA4BA,MAAM,aAAa;;;;;;;;;;;AAYnB,SAAgB,oBAA4B;AAC1C,QAAO,mBAAmB,sBAAsB,CAAC;;;;;;;;;;;;;AAcnD,SAAgB,qBAA6B;AAC3C,QAAO,mBAAmB,uBAAuB,CAAC;;;;;;;;;;AAWpD,SAAgB,gBAA4B;CAC1C,MAAM,cAAc,uBAAuB;AAE3C,KAAI,YAAY,WAAW,WAAW,CACpC,QAAO,gBAAgB,YAAY;AAGrC,QAAO,kBAAkB,YAAY;;;;;;;;;;;;;;;AAgBvC,SAAgB,YAAY;AAC1B,QAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,QAAQ,CAAC;;;;;;;;;;;;;AAc1C,eAAsB,iBAAoC;CACxD,MAAM,cAAc,sBAAsB;AAC1C,KAAI,YAAY,WAAW,WAAW,CACpC,QAAO,aAAa,YAAY;AAElC,QAAO,eAAe,KAAK,QAAQ,YAAY,CAAC;;AAGlD,eAAe,eAAe,UAAqC;AAEjE,SADgB,MAAM,QAAQ,UAAU,EAAE,eAAe,MAAM,CAAC,EAE7D,QAAQ,UAAU,MAAM,QAAQ,CAAC,CACjC,KAAK,UAAU,MAAM,KAAK,CAC1B,MAAM;;AAGX,eAAe,aAAa,KAAgC;CAC1D,MAAM,EAAE,QAAQ,WAAW,YAAY,IAAI;;;;;;CAO3C,MAAM,aAAa,SAAS,GAAG,OAAO,KAAK;CAC3C,MAAM,WAAW,aAAa,WAAW,SAAS;CAGlD,MAAM,CAAC,SAAS,OADA,MAAM,kBAAkB,EACV,OAAO,OAAO,CAAC,SAAS,EACpD,QAAQ,YACT,CAAC;AAEF,QAAO,MACJ,KAAK,SAAS,KAAK,KAAK,MAAM,SAAS,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,KAAK,SAAS,IAAI,CAAC,CACxD,MAAM;;AAGX,SAAS,mBAAmB,aAA6B;AACvD,KAAI,YAAY,WAAW,WAAW,EAAE;;;;;;EAMtC,MAAM,EAAE,QAAQ,WAAW,YAAY,YAAY;AACnD,SAAO,SACH,GAAG,aAAa,OAAO,GAAG,WAC1B,GAAG,aAAa;;AAEtB,QAAO,KAAK,QAAQ,YAAY;;AAGlC,SAAS,uBAA+B;CACtC,MAAM,cAAc,QAAQ,IAAI;AAEhC,KAAI,CAAC,YACH,OAAM,IAAI,MACR,yMAID;AAGH,QAAO;;AAGT,SAAS,wBAAgC;CACvC,MAAM,cAAc,QAAQ,IAAI;AAEhC,KAAI,CAAC,YACH,OAAM,IAAI,MACR,6MAID;AAGH,QAAO;;AAGT,SAAS,kBAAkB,UAA8B;CACvD,MAAM,eAAe,KAAK,QAAQ,SAAS;;CAG3C,eAAe,MACb,cACA,SACiB;EACjB,MAAM,WAAW,KAAK,KAAK,cAAc,aAAa;AAEtD,QAAM,MAAM,KAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,MAAM,CAAC;AACxD,QAAM,UAAU,UAAU,QAAQ;AAElC,UAAQ,KAAK,iBAAiB,WAAW;AACzC,SAAO;;AAGT,QAAO;EACL,MAAM,UAAU,cAAc,MAAM;AAMlC,UAAO,MAJe,gBACpB,qBAAqB,aAAa,EAClC,QACD,EAC2B,WAAW,KAAK,CAAC;;EAE/C,MAAM,UAAU,cAAc,SAAS;AACrC,UAAO,MAAM,qBAAqB,aAAa,EAAE,QAAQ;;EAE3D,MAAM,YAAY,cAAc,SAAS;AACvC,UAAO,MAAM,qBAAqB,aAAa,EAAE,QAAQ;;EAE5D;;AAQH,SAAS,gBAAgB,KAAyB;CAChD,MAAM,SAAS,YAAY,IAAI;;CAG/B,eAAe,MACb,cACA,SACA,aACiB;EACjB,MAAM,aAAa,YAAY,OAAO,QAAQ,aAAa;EAC3D,MAAM,UAAU,GAAG,aAAa,OAAO,OAAO,GAAG;EAGjD,MAAM,QADU,MAAM,kBAAkB,EACnB,OAAO,OAAO,OAAO,CAAC,KAAK,WAAW;EAE3D,MAAM,OACJ,OAAO,YAAY,YAAY,OAAO,SAAS,QAAQ,GACnD,UACA,OAAO,KAAK,QAAQ;AAE1B,QAAM,KAAK,KAAK,MAAM;GACpB;GACA,WAAW;GACZ,CAAC;AAEF,UAAQ,KAAK,iBAAiB,UAAU;AACxC,SAAO;;AAGT,QAAO;EACL,MAAM,UAAU,cAAc,MAAM;AAMlC,UAAO,MAJe,gBACpB,qBAAqB,aAAa,EAClC,QACD,EAC2B,WAAW,KAAK,EAAE,mBAAmB;;EAEnE,MAAM,UAAU,cAAc,SAAS;GACrC,MAAM,OAAO,qBAAqB,aAAa;AAC/C,UAAO,MAAM,MAAM,SAAS,eAAe,KAAK,CAAC;;EAEnD,MAAM,YAAY,cAAc,SAAS;AACvC,UAAO,MACL,qBAAqB,aAAa,EAClC,SACA,2BACD;;EAEJ;;;;;;AAOH,IAAI,gBAAgC;AACpC,eAAe,mBAAqC;AAClD,KAAI,cAAe,QAAO;CAC1B,MAAM,EAAE,SAAS,gBAAgB,MAAM,OAAO;AAC9C,iBAAgB,IAAI,aAAa;AACjC,QAAO;;AAGT,SAAS,YAAY,KAAwB;CAC3C,MAAM,gBAAgB,IAAI,MAAM,EAAkB;CAClD,MAAM,aAAa,cAAc,QAAQ,IAAI;AAE7C,KAAI,eAAe,IAAI;AACrB,MAAI,CAAC,cACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,yBAAyB;AAEpE,SAAO;GAAE,QAAQ;GAAe,QAAQ;GAAI;;CAG9C,MAAM,SAAS,cAAc,MAAM,GAAG,WAAW;CACjD,MAAM,SAAS,cAAc,MAAM,aAAa,EAAE,CAAC,QAAQ,QAAQ,GAAG;AAEtE,KAAI,CAAC,OACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,yBAAyB;AAGpE,QAAO;EAAE;EAAQ;EAAQ;;AAG3B,SAAS,YAAY,QAAgB,UAA0B;CAC7D,MAAM,qBAAqB,SAAS,QAAQ,QAAQ,GAAG;AACvD,QAAO,SAAS,GAAG,OAAO,GAAG,uBAAuB;;AAGtD,SAAS,qBAAqB,cAA8B;AAC1D,KAAI,CAAC,gBAAgB,aAAa,MAAM,KAAK,GAC3C,OAAM,IAAI,MAAM,qBAAqB;AAGvC,KAAI,KAAK,WAAW,aAAa,IAAI,aAAa,WAAW,IAAI,CAC/D,OAAM,IAAI,MACR,oCAAoC,aAAa,oCAElD;CAGH,MAAM,aAAa,KAAK,MAAM,UAAU,aAAa,QAAQ,OAAO,IAAI,CAAC;;;;;;AAOzE,KAAI,WAAW,MAAM,IAAI,CAAC,SAAS,KAAK,CACtC,OAAM,IAAI,MACR,4CAA4C,aAAa,IAC1D;AAGH,QAAO;;AAGT,SAAS,gBAAgB,cAAsB,WAA2B;AACxE,QAAO,aAAa,SAAS,UAAU,GACnC,eACA,GAAG,eAAe;;AAGxB,SAAS,WAAW,MAAuB;AACzC,QAAO,GAAG,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC;;AAG1C,MAAM,qBAA6C;CACjD,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,OAAO;CACR;AAED,SAAS,eAAe,cAA8B;AAEpD,QAAO,mBADK,KAAK,MAAM,QAAQ,aAAa,CAAC,aAAa,KACxB"}
+{"version":3,"file":"files.mjs","names":[],"sources":["../src/files.ts"],"sourcesContent":["import { mkdir, readdir, readFile, writeFile } from \"node:fs/promises\";\nimport path from \"node:path\";\nimport { consola } from \"consola\";\nimport { z } from \"zod\";\nimport type { Storage } from \"@google-cloud/storage\";\n\n/**\n * Writer for job output files, produced by `getFileWriter()`. A single\n * writer instance targets the destination configured via the\n * `outputFilesPath` runner option (local directory or `gs://` URI). All\n * methods return the resolved absolute path or `gs://` URI of the written\n * file.\n */\nexport interface FileWriter {\n  /**\n   * Write a value as pretty-printed JSON (2-space indent, trailing newline).\n   * The `.json` extension is added if missing.\n   */\n  writeJson(relativePath: string, data: unknown): Promise<string>;\n  /** Write a UTF-8 string (e.g. CSV, SVG, plain text). */\n  writeText(relativePath: string, content: string): Promise<string>;\n  /** Write a binary buffer. */\n  writeBuffer(\n    relativePath: string,\n    content: Buffer | Uint8Array,\n  ): Promise<string>;\n}\n\nconst GCS_PREFIX = \"gs://\";\n\n/**\n * Return the resolved input files destination — where the job reads from.\n * Local paths are resolved to an absolute filesystem path; `gs://` URIs\n * are validated and canonicalized (trailing slashes stripped). Use this\n * to locate fixtures, reference datasets, or files produced by another\n * job, and read them with `node:fs` or `@google-cloud/storage` directly.\n *\n * Throws when no input destination is configured or the configured\n * `gs://` URI is malformed (e.g. missing bucket).\n */\nexport function getInputFilesPath(): string {\n  return resolveDestination(readInputDestination());\n}\n\n/**\n * Return the resolved output files destination — where `getFileWriter()`\n * writes to. Local paths are resolved to an absolute filesystem path;\n * `gs://` URIs are validated and canonicalized (trailing slashes\n * stripped). Useful when a job needs to read back its own artifacts\n * (e.g., to chain steps within a handler) or pass the destination to\n * another tool.\n *\n * Throws when no output destination is configured or the configured\n * `gs://` URI is malformed (e.g. missing bucket).\n */\nexport function getOutputFilesPath(): string {\n  return resolveDestination(readOutputDestination());\n}\n\n/**\n * Return a writer that persists files to the destination configured via\n * the `outputFilesPath` runner option. Local paths are used for local\n * execution; `gs://bucket[/prefix]` URIs are used for Cloud Run\n * deployments.\n *\n * Throws when no output destination is configured.\n */\nexport function getFileWriter(): FileWriter {\n  const destination = readOutputDestination();\n\n  if (destination.startsWith(GCS_PREFIX)) {\n    return createGcsWriter(destination);\n  }\n\n  return createLocalWriter(destination);\n}\n\n/**\n * Mark a Zod string field as a file input. Interactive mode detects this\n * marker and offers a selectable list of files from `getInputFilesPath()`\n * instead of a free-text prompt.\n *\n * ```ts\n * z.object({\n *   file: fileInput().describe(\"CSV to process\"),\n * });\n * ```\n *\n * Chains with `.describe()`, `.optional()`, `.default()` as usual.\n */\nexport function fileInput() {\n  return z.string().meta({ kind: \"file\" });\n}\n\n/**\n * List filenames available under the configured input files destination.\n * Returns names relative to the destination directory (no leading slash).\n * Results are sorted alphabetically.\n *\n * Local destinations list only top-level files (nested directories are\n * skipped for now). `gs://` destinations list every object under the\n * configured prefix.\n *\n * Throws when no input destination is configured.\n */\nexport async function listInputFiles(): Promise<string[]> {\n  const destination = readInputDestination();\n  if (destination.startsWith(GCS_PREFIX)) {\n    return listGcsFiles(destination);\n  }\n  return listLocalFiles(path.resolve(destination));\n}\n\n/**\n * Read a file from the configured input files destination as raw bytes.\n * Local paths and `gs://` URIs are handled transparently.\n *\n * `relativePath` is sanitized the same way the writer sanitizes output\n * paths — absolute paths and upward traversal (`..`) are rejected so a\n * job can't escape its configured destination.\n *\n * Throws when no input destination is configured.\n */\nexport async function readInputBuffer(relativePath: string): Promise<Buffer> {\n  return readInputRaw(relativePath);\n}\n\n/**\n * Read a UTF-8 text file from the configured input files destination.\n * Use this for CSV, JSON, plain text, SVG — anything character-based.\n *\n * See `readInputBuffer()` for path-sanitization and configuration\n * semantics.\n */\nexport async function readInputText(relativePath: string): Promise<string> {\n  const buffer = await readInputRaw(relativePath);\n  return buffer.toString(\"utf8\");\n}\n\n/**\n * Read and `JSON.parse` a file from the configured input files\n * destination. Parse errors from a malformed file propagate as the\n * standard `SyntaxError` thrown by `JSON.parse`.\n *\n * See `readInputBuffer()` for path-sanitization and configuration\n * semantics.\n */\nexport async function readInputJson<T = unknown>(\n  relativePath: string,\n): Promise<T> {\n  const text = await readInputText(relativePath);\n  return JSON.parse(text) as T;\n}\n\nasync function readInputRaw(relativePath: string): Promise<Buffer> {\n  const destination = readInputDestination();\n  const safeRelative = sanitizeRelativePath(relativePath);\n\n  if (destination.startsWith(GCS_PREFIX)) {\n    return readGcsObject(destination, safeRelative);\n  }\n\n  return readFile(path.join(path.resolve(destination), safeRelative));\n}\n\nasync function readGcsObject(\n  uri: string,\n  safeRelative: string,\n): Promise<Buffer> {\n  const { bucket, prefix } = parseGcsUri(uri);\n  const storage = await getStorageClient();\n  const [buf] = await storage\n    .bucket(bucket)\n    .file(joinGcsPath(prefix, safeRelative))\n    .download();\n  return buf;\n}\n\nasync function listLocalFiles(basePath: string): Promise<string[]> {\n  const entries = await readdir(basePath, { withFileTypes: true });\n  return entries\n    .filter((entry) => entry.isFile())\n    .map((entry) => entry.name)\n    .sort();\n}\n\nasync function listGcsFiles(uri: string): Promise<string[]> {\n  const { bucket, prefix } = parseGcsUri(uri);\n\n  /**\n   * Terminate the list prefix with `/` so GCS treats it as a folder and\n   * not a bare substring — otherwise a configured `inputs` would also\n   * match sibling keys like `inputs2/…` or `inputs-backup/…`.\n   */\n  const listPrefix = prefix ? `${prefix}/` : undefined;\n  const stripLen = listPrefix ? listPrefix.length : 0;\n\n  const storage = await getStorageClient();\n  const [files] = await storage.bucket(bucket).getFiles({\n    prefix: listPrefix,\n  });\n\n  return files\n    .map((file) => file.name.slice(stripLen))\n    .filter((name) => name.length > 0 && !name.endsWith(\"/\"))\n    .sort();\n}\n\nfunction resolveDestination(destination: string): string {\n  if (destination.startsWith(GCS_PREFIX)) {\n    /**\n     * Validate and canonicalize via the same parser the writer uses, so\n     * read and write paths stay consistent — reject `gs://` without a\n     * bucket and strip any trailing slash from the prefix.\n     */\n    const { bucket, prefix } = parseGcsUri(destination);\n    return prefix\n      ? `${GCS_PREFIX}${bucket}/${prefix}`\n      : `${GCS_PREFIX}${bucket}`;\n  }\n  return path.resolve(destination);\n}\n\nfunction readInputDestination(): string {\n  const destination = process.env.JOB_INPUT_FILES_PATH;\n\n  if (!destination) {\n    throw new Error(\n      \"No input files destination configured.\\n\" +\n        \"Set `localInputFilesPath` or the current environment's \" +\n        \"`inputFilesPath` in your job-runner.config.ts, or set \" +\n        \"JOB_INPUT_FILES_PATH directly in the environment.\",\n    );\n  }\n\n  return destination;\n}\n\nfunction readOutputDestination(): string {\n  const destination = process.env.JOB_OUTPUT_FILES_PATH;\n\n  if (!destination) {\n    throw new Error(\n      \"No output files destination configured.\\n\" +\n        \"Set `localOutputFilesPath` or the current environment's \" +\n        \"`outputFilesPath` in your job-runner.config.ts, or set \" +\n        \"JOB_OUTPUT_FILES_PATH directly in the environment.\",\n    );\n  }\n\n  return destination;\n}\n\nfunction createLocalWriter(basePath: string): FileWriter {\n  const resolvedBase = path.resolve(basePath);\n\n  /** Takes an already-sanitized relative path and writes the content. */\n  async function write(\n    safeRelative: string,\n    content: string | Uint8Array,\n  ): Promise<string> {\n    const fullPath = path.join(resolvedBase, safeRelative);\n\n    await mkdir(path.dirname(fullPath), { recursive: true });\n    await writeFile(fullPath, content);\n\n    consola.info(`File written: ${fullPath}`);\n    return fullPath;\n  }\n\n  return {\n    async writeJson(relativePath, data) {\n      /** Sanitize before appending the extension so \"\" doesn't become \".json\". */\n      const withExtension = ensureExtension(\n        sanitizeRelativePath(relativePath),\n        \".json\",\n      );\n      return write(withExtension, formatJson(data));\n    },\n    async writeText(relativePath, content) {\n      return write(sanitizeRelativePath(relativePath), content);\n    },\n    async writeBuffer(relativePath, content) {\n      return write(sanitizeRelativePath(relativePath), content);\n    },\n  };\n}\n\ninterface GcsTarget {\n  bucket: string;\n  prefix: string;\n}\n\nfunction createGcsWriter(uri: string): FileWriter {\n  const target = parseGcsUri(uri);\n\n  /** Takes an already-sanitized relative path and uploads the content. */\n  async function write(\n    safeRelative: string,\n    content: string | Buffer | Uint8Array,\n    contentType: string,\n  ): Promise<string> {\n    const objectName = joinGcsPath(target.prefix, safeRelative);\n    const fullUri = `${GCS_PREFIX}${target.bucket}/${objectName}`;\n\n    const storage = await getStorageClient();\n    const file = storage.bucket(target.bucket).file(objectName);\n\n    const body =\n      typeof content === \"string\" || Buffer.isBuffer(content)\n        ? content\n        : Buffer.from(content);\n\n    await file.save(body, {\n      contentType,\n      resumable: false,\n    });\n\n    consola.info(`File written: ${fullUri}`);\n    return fullUri;\n  }\n\n  return {\n    async writeJson(relativePath, data) {\n      /** Sanitize before appending the extension so \"\" doesn't become \".json\". */\n      const withExtension = ensureExtension(\n        sanitizeRelativePath(relativePath),\n        \".json\",\n      );\n      return write(withExtension, formatJson(data), \"application/json\");\n    },\n    async writeText(relativePath, content) {\n      const safe = sanitizeRelativePath(relativePath);\n      return write(safe, content, contentTypeFor(safe));\n    },\n    async writeBuffer(relativePath, content) {\n      return write(\n        sanitizeRelativePath(relativePath),\n        content,\n        \"application/octet-stream\",\n      );\n    },\n  };\n}\n\n/**\n * Lazy Storage client singleton. The `@google-cloud/storage` module is only\n * loaded when a `gs://` destination is actually used for a write.\n */\nlet storageClient: Storage | null = null;\nasync function getStorageClient(): Promise<Storage> {\n  if (storageClient) return storageClient;\n  const { Storage: StorageCtor } = await import(\"@google-cloud/storage\");\n  storageClient = new StorageCtor();\n  return storageClient;\n}\n\nfunction parseGcsUri(uri: string): GcsTarget {\n  const withoutScheme = uri.slice(GCS_PREFIX.length);\n  const slashIndex = withoutScheme.indexOf(\"/\");\n\n  if (slashIndex === -1) {\n    if (!withoutScheme) {\n      throw new Error(`Invalid GCS URI: \"${uri}\" (missing bucket name)`);\n    }\n    return { bucket: withoutScheme, prefix: \"\" };\n  }\n\n  const bucket = withoutScheme.slice(0, slashIndex);\n  const prefix = withoutScheme.slice(slashIndex + 1).replace(/\\/+$/, \"\");\n\n  if (!bucket) {\n    throw new Error(`Invalid GCS URI: \"${uri}\" (missing bucket name)`);\n  }\n\n  return { bucket, prefix };\n}\n\nfunction joinGcsPath(prefix: string, relative: string): string {\n  const normalizedRelative = relative.replace(/^\\/+/, \"\");\n  return prefix ? `${prefix}/${normalizedRelative}` : normalizedRelative;\n}\n\nfunction sanitizeRelativePath(relativePath: string): string {\n  if (!relativePath || relativePath.trim() === \"\") {\n    throw new Error(\"File path is empty\");\n  }\n\n  if (path.isAbsolute(relativePath) || relativePath.startsWith(\"/\")) {\n    throw new Error(\n      `File path must be relative, got \"${relativePath}\". ` +\n        \"Absolute paths are not allowed.\",\n    );\n  }\n\n  const normalized = path.posix.normalize(relativePath.replace(/\\\\/g, \"/\"));\n\n  /**\n   * Reject `..` only as a distinct path segment, not as a prefix. A filename\n   * like \"..hidden\" is a legitimate dotfile variant, while \"..\" / \"../x\" /\n   * \"a/../../x\" all produce a `..` segment after normalization.\n   */\n  if (normalized.split(\"/\").includes(\"..\")) {\n    throw new Error(\n      `File path must not traverse upward, got \"${relativePath}\".`,\n    );\n  }\n\n  return normalized;\n}\n\nfunction ensureExtension(relativePath: string, extension: string): string {\n  return relativePath.endsWith(extension)\n    ? relativePath\n    : `${relativePath}${extension}`;\n}\n\nfunction formatJson(data: unknown): string {\n  return `${JSON.stringify(data, null, 2)}\\n`;\n}\n\nconst TEXT_CONTENT_TYPES: Record<string, string> = {\n  \".csv\": \"text/csv; charset=utf-8\",\n  \".json\": \"application/json\",\n  \".svg\": \"image/svg+xml\",\n  \".txt\": \"text/plain; charset=utf-8\",\n  \".html\": \"text/html; charset=utf-8\",\n  \".xml\": \"application/xml\",\n  \".yaml\": \"application/yaml\",\n  \".yml\": \"application/yaml\",\n  \".md\": \"text/markdown; charset=utf-8\",\n};\n\nfunction contentTypeFor(relativePath: string): string {\n  const ext = path.posix.extname(relativePath).toLowerCase();\n  return TEXT_CONTENT_TYPES[ext] ?? \"text/plain; charset=utf-8\";\n}\n"],"mappings":";;;;;;AA4BA,MAAM,aAAa;;;;;;;;;;;AAYnB,SAAgB,oBAA4B;AAC1C,QAAO,mBAAmB,sBAAsB,CAAC;;;;;;;;;;;;;AAcnD,SAAgB,qBAA6B;AAC3C,QAAO,mBAAmB,uBAAuB,CAAC;;;;;;;;;;AAWpD,SAAgB,gBAA4B;CAC1C,MAAM,cAAc,uBAAuB;AAE3C,KAAI,YAAY,WAAW,WAAW,CACpC,QAAO,gBAAgB,YAAY;AAGrC,QAAO,kBAAkB,YAAY;;;;;;;;;;;;;;;AAgBvC,SAAgB,YAAY;AAC1B,QAAO,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,QAAQ,CAAC;;;;;;;;;;;;;AAc1C,eAAsB,iBAAoC;CACxD,MAAM,cAAc,sBAAsB;AAC1C,KAAI,YAAY,WAAW,WAAW,CACpC,QAAO,aAAa,YAAY;AAElC,QAAO,eAAe,KAAK,QAAQ,YAAY,CAAC;;;;;;;;;;;;AAalD,eAAsB,gBAAgB,cAAuC;AAC3E,QAAO,aAAa,aAAa;;;;;;;;;AAUnC,eAAsB,cAAc,cAAuC;AAEzE,SADe,MAAM,aAAa,aAAa,EACjC,SAAS,OAAO;;;;;;;;;;AAWhC,eAAsB,cACpB,cACY;CACZ,MAAM,OAAO,MAAM,cAAc,aAAa;AAC9C,QAAO,KAAK,MAAM,KAAK;;AAGzB,eAAe,aAAa,cAAuC;CACjE,MAAM,cAAc,sBAAsB;CAC1C,MAAM,eAAe,qBAAqB,aAAa;AAEvD,KAAI,YAAY,WAAW,WAAW,CACpC,QAAO,cAAc,aAAa,aAAa;AAGjD,QAAO,SAAS,KAAK,KAAK,KAAK,QAAQ,YAAY,EAAE,aAAa,CAAC;;AAGrE,eAAe,cACb,KACA,cACiB;CACjB,MAAM,EAAE,QAAQ,WAAW,YAAY,IAAI;CAE3C,MAAM,CAAC,OAAO,OADE,MAAM,kBAAkB,EAErC,OAAO,OAAO,CACd,KAAK,YAAY,QAAQ,aAAa,CAAC,CACvC,UAAU;AACb,QAAO;;AAGT,eAAe,eAAe,UAAqC;AAEjE,SADgB,MAAM,QAAQ,UAAU,EAAE,eAAe,MAAM,CAAC,EAE7D,QAAQ,UAAU,MAAM,QAAQ,CAAC,CACjC,KAAK,UAAU,MAAM,KAAK,CAC1B,MAAM;;AAGX,eAAe,aAAa,KAAgC;CAC1D,MAAM,EAAE,QAAQ,WAAW,YAAY,IAAI;;;;;;CAO3C,MAAM,aAAa,SAAS,GAAG,OAAO,KAAK;CAC3C,MAAM,WAAW,aAAa,WAAW,SAAS;CAGlD,MAAM,CAAC,SAAS,OADA,MAAM,kBAAkB,EACV,OAAO,OAAO,CAAC,SAAS,EACpD,QAAQ,YACT,CAAC;AAEF,QAAO,MACJ,KAAK,SAAS,KAAK,KAAK,MAAM,SAAS,CAAC,CACxC,QAAQ,SAAS,KAAK,SAAS,KAAK,CAAC,KAAK,SAAS,IAAI,CAAC,CACxD,MAAM;;AAGX,SAAS,mBAAmB,aAA6B;AACvD,KAAI,YAAY,WAAW,WAAW,EAAE;;;;;;EAMtC,MAAM,EAAE,QAAQ,WAAW,YAAY,YAAY;AACnD,SAAO,SACH,GAAG,aAAa,OAAO,GAAG,WAC1B,GAAG,aAAa;;AAEtB,QAAO,KAAK,QAAQ,YAAY;;AAGlC,SAAS,uBAA+B;CACtC,MAAM,cAAc,QAAQ,IAAI;AAEhC,KAAI,CAAC,YACH,OAAM,IAAI,MACR,yMAID;AAGH,QAAO;;AAGT,SAAS,wBAAgC;CACvC,MAAM,cAAc,QAAQ,IAAI;AAEhC,KAAI,CAAC,YACH,OAAM,IAAI,MACR,6MAID;AAGH,QAAO;;AAGT,SAAS,kBAAkB,UAA8B;CACvD,MAAM,eAAe,KAAK,QAAQ,SAAS;;CAG3C,eAAe,MACb,cACA,SACiB;EACjB,MAAM,WAAW,KAAK,KAAK,cAAc,aAAa;AAEtD,QAAM,MAAM,KAAK,QAAQ,SAAS,EAAE,EAAE,WAAW,MAAM,CAAC;AACxD,QAAM,UAAU,UAAU,QAAQ;AAElC,UAAQ,KAAK,iBAAiB,WAAW;AACzC,SAAO;;AAGT,QAAO;EACL,MAAM,UAAU,cAAc,MAAM;AAMlC,UAAO,MAJe,gBACpB,qBAAqB,aAAa,EAClC,QACD,EAC2B,WAAW,KAAK,CAAC;;EAE/C,MAAM,UAAU,cAAc,SAAS;AACrC,UAAO,MAAM,qBAAqB,aAAa,EAAE,QAAQ;;EAE3D,MAAM,YAAY,cAAc,SAAS;AACvC,UAAO,MAAM,qBAAqB,aAAa,EAAE,QAAQ;;EAE5D;;AAQH,SAAS,gBAAgB,KAAyB;CAChD,MAAM,SAAS,YAAY,IAAI;;CAG/B,eAAe,MACb,cACA,SACA,aACiB;EACjB,MAAM,aAAa,YAAY,OAAO,QAAQ,aAAa;EAC3D,MAAM,UAAU,GAAG,aAAa,OAAO,OAAO,GAAG;EAGjD,MAAM,QADU,MAAM,kBAAkB,EACnB,OAAO,OAAO,OAAO,CAAC,KAAK,WAAW;EAE3D,MAAM,OACJ,OAAO,YAAY,YAAY,OAAO,SAAS,QAAQ,GACnD,UACA,OAAO,KAAK,QAAQ;AAE1B,QAAM,KAAK,KAAK,MAAM;GACpB;GACA,WAAW;GACZ,CAAC;AAEF,UAAQ,KAAK,iBAAiB,UAAU;AACxC,SAAO;;AAGT,QAAO;EACL,MAAM,UAAU,cAAc,MAAM;AAMlC,UAAO,MAJe,gBACpB,qBAAqB,aAAa,EAClC,QACD,EAC2B,WAAW,KAAK,EAAE,mBAAmB;;EAEnE,MAAM,UAAU,cAAc,SAAS;GACrC,MAAM,OAAO,qBAAqB,aAAa;AAC/C,UAAO,MAAM,MAAM,SAAS,eAAe,KAAK,CAAC;;EAEnD,MAAM,YAAY,cAAc,SAAS;AACvC,UAAO,MACL,qBAAqB,aAAa,EAClC,SACA,2BACD;;EAEJ;;;;;;AAOH,IAAI,gBAAgC;AACpC,eAAe,mBAAqC;AAClD,KAAI,cAAe,QAAO;CAC1B,MAAM,EAAE,SAAS,gBAAgB,MAAM,OAAO;AAC9C,iBAAgB,IAAI,aAAa;AACjC,QAAO;;AAGT,SAAS,YAAY,KAAwB;CAC3C,MAAM,gBAAgB,IAAI,MAAM,EAAkB;CAClD,MAAM,aAAa,cAAc,QAAQ,IAAI;AAE7C,KAAI,eAAe,IAAI;AACrB,MAAI,CAAC,cACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,yBAAyB;AAEpE,SAAO;GAAE,QAAQ;GAAe,QAAQ;GAAI;;CAG9C,MAAM,SAAS,cAAc,MAAM,GAAG,WAAW;CACjD,MAAM,SAAS,cAAc,MAAM,aAAa,EAAE,CAAC,QAAQ,QAAQ,GAAG;AAEtE,KAAI,CAAC,OACH,OAAM,IAAI,MAAM,qBAAqB,IAAI,yBAAyB;AAGpE,QAAO;EAAE;EAAQ;EAAQ;;AAG3B,SAAS,YAAY,QAAgB,UAA0B;CAC7D,MAAM,qBAAqB,SAAS,QAAQ,QAAQ,GAAG;AACvD,QAAO,SAAS,GAAG,OAAO,GAAG,uBAAuB;;AAGtD,SAAS,qBAAqB,cAA8B;AAC1D,KAAI,CAAC,gBAAgB,aAAa,MAAM,KAAK,GAC3C,OAAM,IAAI,MAAM,qBAAqB;AAGvC,KAAI,KAAK,WAAW,aAAa,IAAI,aAAa,WAAW,IAAI,CAC/D,OAAM,IAAI,MACR,oCAAoC,aAAa,oCAElD;CAGH,MAAM,aAAa,KAAK,MAAM,UAAU,aAAa,QAAQ,OAAO,IAAI,CAAC;;;;;;AAOzE,KAAI,WAAW,MAAM,IAAI,CAAC,SAAS,KAAK,CACtC,OAAM,IAAI,MACR,4CAA4C,aAAa,IAC1D;AAGH,QAAO;;AAGT,SAAS,gBAAgB,cAAsB,WAA2B;AACxE,QAAO,aAAa,SAAS,UAAU,GACnC,eACA,GAAG,eAAe;;AAGxB,SAAS,WAAW,MAAuB;AACzC,QAAO,GAAG,KAAK,UAAU,MAAM,MAAM,EAAE,CAAC;;AAG1C,MAAM,qBAA6C;CACjD,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,SAAS;CACT,QAAQ;CACR,OAAO;CACR;AAED,SAAS,eAAe,cAA8B;AAEpD,QAAO,mBADK,KAAK,MAAM,QAAQ,aAAa,CAAC,aAAa,KACxB"}

package/dist/index.d.mts

@@ -2,8 +2,8 @@ import { CloudConfig, CloudNetworkConfig, CloudResources, RunnerConfig, RunnerEn
 import { FlagAliases, JobFunction, JobInfo, JobMetadata, JobOptions, RunJobOptions } from "./types.mjs";
 import { defineJob } from "./define-job.mjs";
 import { discoverJobs } from "./discover-jobs.mjs";
-import { FileWriter, fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles } from "./files.mjs";
+import { FileWriter, fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles, readInputBuffer, readInputJson, readInputText } from "./files.mjs";
 import { FieldInfo, extractFieldInfo, formatZodError, generateSchemaHelp, schemaToParseArgsOptions } from "./help.mjs";
 import { TaskContext, getTaskContext } from "./task-context.mjs";
 import { runJob } from "./run-job.mjs";
-export { type CloudConfig, type CloudNetworkConfig, type CloudResources, type FieldInfo, type FileWriter, type FlagAliases, type JobFunction, type JobInfo, type JobMetadata, type JobOptions, type RunJobOptions, type RunnerConfig, type RunnerEnvOptions, type TaskContext, defineJob, defineRunnerConfig, defineRunnerEnv, discoverJobs, extractFieldInfo, fileInput, formatZodError, generateSchemaHelp, getFileWriter, getInputFilesPath, getOutputFilesPath, getTaskContext, listInputFiles, runJob, schemaToParseArgsOptions };
+export { type CloudConfig, type CloudNetworkConfig, type CloudResources, type FieldInfo, type FileWriter, type FlagAliases, type JobFunction, type JobInfo, type JobMetadata, type JobOptions, type RunJobOptions, type RunnerConfig, type RunnerEnvOptions, type TaskContext, defineJob, defineRunnerConfig, defineRunnerEnv, discoverJobs, extractFieldInfo, fileInput, formatZodError, generateSchemaHelp, getFileWriter, getInputFilesPath, getOutputFilesPath, getTaskContext, listInputFiles, readInputBuffer, readInputJson, readInputText, runJob, schemaToParseArgsOptions };

package/dist/index.mjs

@@ -2,8 +2,8 @@ import { defineRunnerConfig, defineRunnerEnv } from "./config.mjs";
 import { extractFieldInfo, formatZodError, generateSchemaHelp, schemaToParseArgsOptions } from "./help.mjs";
 import { defineJob } from "./define-job.mjs";
 import { discoverJobs } from "./discover-jobs.mjs";
-import { fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles } from "./files.mjs";
+import { fileInput, getFileWriter, getInputFilesPath, getOutputFilesPath, listInputFiles, readInputBuffer, readInputJson, readInputText } from "./files.mjs";
 import { getTaskContext } from "./task-context.mjs";
 import { runJob } from "./run-job.mjs";
 
-export { defineJob, defineRunnerConfig, defineRunnerEnv, discoverJobs, extractFieldInfo, fileInput, formatZodError, generateSchemaHelp, getFileWriter, getInputFilesPath, getOutputFilesPath, getTaskContext, listInputFiles, runJob, schemaToParseArgsOptions };
+export { defineJob, defineRunnerConfig, defineRunnerEnv, discoverJobs, extractFieldInfo, fileInput, formatZodError, generateSchemaHelp, getFileWriter, getInputFilesPath, getOutputFilesPath, getTaskContext, listInputFiles, readInputBuffer, readInputJson, readInputText, runJob, schemaToParseArgsOptions };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gcp-job-runner",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Run schema-driven Cloud Run jobs seamlessly in any environment",
   "license": "MIT",
   "repository": {
@@ -37,7 +37,7 @@
     "docs:dev": "vitepress dev docs",
     "docs:build": "vitepress build docs",
     "docs:preview": "vitepress preview docs",
-    "prepare": "husky && tsdown"
+    "prepare": "lefthook install && tsdown"
   },
   "dependencies": {
     "@google-cloud/logging": "^11.2.0",
@@ -52,8 +52,7 @@
     "@codecompose/typescript-config": "3.0.0",
     "@types/node": "^25.3.0",
     "del-cli": "^7.0.0",
-    "husky": "^9.1.7",
-    "lint-staged": "^16.2.7",
+    "lefthook": "^2.1.6",
     "oxfmt": "^0.34.0",
     "oxlint": "^1.49.0",
     "tsdown": "^0.20.1",
@@ -61,15 +60,6 @@
     "vitepress": "^1.6.4",
     "vitest": "^4.0.18"
   },
-  "lint-staged": {
-    "*.{js,jsx,ts,tsx,mjs,cjs}": [
-      "oxfmt",
-      "oxlint -c .oxlintrc.json --import-plugin"
-    ],
-    "*.{json,yaml,yml}": [
-      "oxfmt"
-    ]
-  },
   "engines": {
     "node": ">=22.6.0"
   },

package/src/files.test.ts

@@ -9,6 +9,9 @@ import {
   getInputFilesPath,
   getOutputFilesPath,
   listInputFiles,
+  readInputBuffer,
+  readInputJson,
+  readInputText,
 } from "./files";
 
 describe("getFileWriter", () => {
@@ -471,3 +474,184 @@ describe("listInputFiles (gcs)", () => {
     expect(result).toEqual(["real.txt"]);
   });
 });
+
+describe("readInputText / readInputJson / readInputBuffer (local)", () => {
+  let tempDir: string;
+  const originalInputPath = process.env.JOB_INPUT_FILES_PATH;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(path.join(os.tmpdir(), "read-input-test-"));
+    process.env.JOB_INPUT_FILES_PATH = tempDir;
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    if (originalInputPath === undefined) {
+      delete process.env.JOB_INPUT_FILES_PATH;
+    } else {
+      process.env.JOB_INPUT_FILES_PATH = originalInputPath;
+    }
+  });
+
+  it("readInputText returns file contents decoded as UTF-8", async () => {
+    writeFileSync(path.join(tempDir, "notes.txt"), "héllo\nworld");
+    expect(await readInputText("notes.txt")).toBe("héllo\nworld");
+  });
+
+  it("readInputJson parses JSON", async () => {
+    writeFileSync(path.join(tempDir, "data.json"), '{"count":42,"ok":true}');
+    expect(await readInputJson("data.json")).toEqual({ count: 42, ok: true });
+  });
+
+  it("readInputJson preserves the generic type parameter", async () => {
+    interface Row {
+      id: number;
+    }
+    writeFileSync(path.join(tempDir, "rows.json"), '[{"id":1},{"id":2}]');
+    const rows = await readInputJson<Row[]>("rows.json");
+    expect(rows.map((row) => row.id)).toEqual([1, 2]);
+  });
+
+  it("readInputJson surfaces parse errors from malformed input", async () => {
+    writeFileSync(path.join(tempDir, "bad.json"), "{not json");
+    await expect(readInputJson("bad.json")).rejects.toThrow(SyntaxError);
+  });
+
+  it("readInputBuffer returns raw bytes unchanged", async () => {
+    const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff]);
+    writeFileSync(path.join(tempDir, "blob.bin"), bytes);
+    const result = await readInputBuffer("blob.bin");
+    expect(result.equals(bytes)).toBe(true);
+  });
+
+  it("resolves nested relative paths under the destination", async () => {
+    await mkdir(path.join(tempDir, "db", "airlines"), { recursive: true });
+    writeFileSync(
+      path.join(tempDir, "db", "airlines", "UA.json"),
+      '{"iata":"UA"}',
+    );
+    expect(await readInputJson("db/airlines/UA.json")).toEqual({ iata: "UA" });
+  });
+
+  it("rejects absolute paths", async () => {
+    await expect(readInputText("/etc/passwd")).rejects.toThrow(
+      /Absolute paths are not allowed/,
+    );
+  });
+
+  it("rejects upward traversal", async () => {
+    await expect(readInputText("../escape.txt")).rejects.toThrow(
+      /must not traverse upward/,
+    );
+    await expect(readInputText("a/../../escape.txt")).rejects.toThrow(
+      /must not traverse upward/,
+    );
+  });
+
+  it("rejects empty paths", async () => {
+    await expect(readInputText("")).rejects.toThrow(/empty/);
+    await expect(readInputJson("")).rejects.toThrow(/empty/);
+    await expect(readInputBuffer("")).rejects.toThrow(/empty/);
+  });
+
+  it("throws the standard unconfigured error when JOB_INPUT_FILES_PATH is unset", async () => {
+    delete process.env.JOB_INPUT_FILES_PATH;
+    await expect(readInputText("notes.txt")).rejects.toThrow(/inputFilesPath/);
+  });
+
+  it("throws the standard unconfigured error when JOB_INPUT_FILES_PATH is empty", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "";
+    await expect(readInputText("notes.txt")).rejects.toThrow(/inputFilesPath/);
+    await expect(readInputJson("notes.txt")).rejects.toThrow(/inputFilesPath/);
+    await expect(readInputBuffer("notes.txt")).rejects.toThrow(
+      /inputFilesPath/,
+    );
+  });
+});
+
+describe("readInputText / readInputJson / readInputBuffer (gcs)", () => {
+  const originalInputPath = process.env.JOB_INPUT_FILES_PATH;
+  const downloadMock = vi.fn();
+  const fileMock = vi.fn(() => ({ download: downloadMock }));
+  const bucketMock = vi.fn(() => ({ file: fileMock }));
+
+  beforeEach(() => {
+    downloadMock.mockReset();
+    fileMock.mockClear();
+    bucketMock.mockClear();
+    vi.doMock("@google-cloud/storage", () => ({
+      Storage: class {
+        bucket = bucketMock;
+      },
+    }));
+  });
+
+  afterEach(() => {
+    vi.doUnmock("@google-cloud/storage");
+    vi.resetModules();
+    if (originalInputPath === undefined) {
+      delete process.env.JOB_INPUT_FILES_PATH;
+    } else {
+      process.env.JOB_INPUT_FILES_PATH = originalInputPath;
+    }
+  });
+
+  it("downloads the object and decodes it as text", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "gs://my-bucket/inputs";
+    downloadMock.mockResolvedValue([Buffer.from("hello cloud", "utf8")]);
+
+    vi.resetModules();
+    const { readInputText: freshText } = await import("./files");
+    const result = await freshText("notes.txt");
+
+    expect(bucketMock).toHaveBeenCalledWith("my-bucket");
+    expect(fileMock).toHaveBeenCalledWith("inputs/notes.txt");
+    expect(result).toBe("hello cloud");
+  });
+
+  it("parses JSON downloaded from the configured bucket", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "gs://my-bucket/inputs";
+    downloadMock.mockResolvedValue([Buffer.from('{"count":7}', "utf8")]);
+
+    vi.resetModules();
+    const { readInputJson: freshJson } = await import("./files");
+    const result = await freshJson<{ count: number }>("report.json");
+
+    expect(fileMock).toHaveBeenCalledWith("inputs/report.json");
+    expect(result).toEqual({ count: 7 });
+  });
+
+  it("returns the raw buffer without decoding", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "gs://my-bucket/inputs";
+    const bytes = Buffer.from([0x00, 0xff, 0x10]);
+    downloadMock.mockResolvedValue([bytes]);
+
+    vi.resetModules();
+    const { readInputBuffer: freshBuffer } = await import("./files");
+    const result = await freshBuffer("blob.bin");
+
+    expect(result.equals(bytes)).toBe(true);
+  });
+
+  it("handles bucket-only URIs (no prefix)", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "gs://my-bucket";
+    downloadMock.mockResolvedValue([Buffer.from("x", "utf8")]);
+
+    vi.resetModules();
+    const { readInputText: freshText } = await import("./files");
+    await freshText("a.txt");
+
+    expect(fileMock).toHaveBeenCalledWith("a.txt");
+  });
+
+  it("rejects traversal before issuing the download", async () => {
+    process.env.JOB_INPUT_FILES_PATH = "gs://my-bucket/inputs";
+
+    vi.resetModules();
+    const { readInputText: freshText } = await import("./files");
+    await expect(freshText("../secret.txt")).rejects.toThrow(
+      /must not traverse upward/,
+    );
+    expect(downloadMock).not.toHaveBeenCalled();
+  });
+});

package/src/files.ts

@@ -1,4 +1,4 @@
-import { mkdir, readdir, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { consola } from "consola";
 import { z } from "zod";
@@ -111,6 +111,71 @@ export async function listInputFiles(): Promise<string[]> {
   return listLocalFiles(path.resolve(destination));
 }
 
+/**
+ * Read a file from the configured input files destination as raw bytes.
+ * Local paths and `gs://` URIs are handled transparently.
+ *
+ * `relativePath` is sanitized the same way the writer sanitizes output
+ * paths — absolute paths and upward traversal (`..`) are rejected so a
+ * job can't escape its configured destination.
+ *
+ * Throws when no input destination is configured.
+ */
+export async function readInputBuffer(relativePath: string): Promise<Buffer> {
+  return readInputRaw(relativePath);
+}
+
+/**
+ * Read a UTF-8 text file from the configured input files destination.
+ * Use this for CSV, JSON, plain text, SVG — anything character-based.
+ *
+ * See `readInputBuffer()` for path-sanitization and configuration
+ * semantics.
+ */
+export async function readInputText(relativePath: string): Promise<string> {
+  const buffer = await readInputRaw(relativePath);
+  return buffer.toString("utf8");
+}
+
+/**
+ * Read and `JSON.parse` a file from the configured input files
+ * destination. Parse errors from a malformed file propagate as the
+ * standard `SyntaxError` thrown by `JSON.parse`.
+ *
+ * See `readInputBuffer()` for path-sanitization and configuration
+ * semantics.
+ */
+export async function readInputJson<T = unknown>(
+  relativePath: string,
+): Promise<T> {
+  const text = await readInputText(relativePath);
+  return JSON.parse(text) as T;
+}
+
+async function readInputRaw(relativePath: string): Promise<Buffer> {
+  const destination = readInputDestination();
+  const safeRelative = sanitizeRelativePath(relativePath);
+
+  if (destination.startsWith(GCS_PREFIX)) {
+    return readGcsObject(destination, safeRelative);
+  }
+
+  return readFile(path.join(path.resolve(destination), safeRelative));
+}
+
+async function readGcsObject(
+  uri: string,
+  safeRelative: string,
+): Promise<Buffer> {
+  const { bucket, prefix } = parseGcsUri(uri);
+  const storage = await getStorageClient();
+  const [buf] = await storage
+    .bucket(bucket)
+    .file(joinGcsPath(prefix, safeRelative))
+    .download();
+  return buf;
+}
+
 async function listLocalFiles(basePath: string): Promise<string[]> {
   const entries = await readdir(basePath, { withFileTypes: true });
   return entries

package/src/index.ts

@@ -7,6 +7,9 @@ export {
   getInputFilesPath,
   getOutputFilesPath,
   listInputFiles,
+  readInputBuffer,
+  readInputJson,
+  readInputText,
 } from "./files";
 export type { FileWriter } from "./files";
 export {