@nyc-transit-kit/cli 0.1.0

package/src/commands/socrata.ts

@@ -0,0 +1,206 @@
+import type { Soda3ExportFormat } from "@nyc-transit-kit/contracts/soda3"
+import { buildExportUrl } from "@nyc-transit-kit/soda3/endpoints"
+import { exportResponse } from "@nyc-transit-kit/soda3/export"
+import { queryRows } from "@nyc-transit-kit/soda3/query"
+import * as Effect from "effect/Effect"
+import * as Command from "effect/unstable/cli/Command"
+import * as Flag from "effect/unstable/cli/Flag"
+import { runEffect, soqlFromSelect, writeSuccess } from "./shared"
+import { writeResponseToFile, writeSoda3ExportDryRun, writeSoda3QueryDryRun } from "./soda3-shared"
+import { CliCommandContext } from "./types"
+
+export const socrataCommandPaths: ReadonlyArray<ReadonlyArray<string>> = [
+  ["socrata", "query"],
+  ["socrata", "export"],
+  ["socrata", "range-probe"]
+]
+
+export const handleSocrataQuery = (config: {
+  readonly domain: string
+  readonly datasetId: string
+  readonly select: string
+  readonly dryRun: boolean
+}) =>
+  Effect.gen(function* () {
+    const context = yield* CliCommandContext
+    const tokens = ["socrata", "query"]
+    const request = {
+      domain: config.domain,
+      datasetId: config.datasetId,
+      query: soqlFromSelect(config.select)
+    }
+
+    if (config.dryRun) {
+      return yield* writeSoda3QueryDryRun({
+        context,
+        apiFamily: "socrata",
+        tokens,
+        ...request
+      })
+    }
+
+    return yield* runEffect(
+      context,
+      "socrata",
+      tokens,
+      queryRows(request).pipe(Effect.provide(context.runtime.soda3Layer))
+    )
+  })
+
+export const handleSocrataExport = (config: {
+  readonly domain: string
+  readonly datasetId: string
+  readonly format: Soda3ExportFormat
+  readonly output: string
+  readonly dryRun: boolean
+}) =>
+  Effect.gen(function* () {
+    const context = yield* CliCommandContext
+    const tokens = ["socrata", "export"]
+    const request = {
+      domain: config.domain,
+      datasetId: config.datasetId,
+      format: config.format
+    }
+
+    if (config.dryRun) {
+      return yield* writeSoda3ExportDryRun({
+        context,
+        apiFamily: "socrata",
+        tokens,
+        output: config.output,
+        ...request
+      })
+    }
+
+    return yield* runEffect(
+      context,
+      "socrata",
+      tokens,
+      Effect.gen(function* () {
+        const response = yield* exportResponse(request).pipe(
+          Effect.provide(context.runtime.soda3Layer)
+        )
+        return yield* writeResponseToFile(config.output, response)
+      })
+    )
+  })
+
+export const handleSocrataRangeProbe = (config: {
+  readonly domain: string
+  readonly datasetId: string
+  readonly format: Soda3ExportFormat
+  readonly rangeEnd: number
+  readonly dryRun: boolean
+}) =>
+  Effect.gen(function* () {
+    const context = yield* CliCommandContext
+    const tokens = ["socrata", "range-probe"]
+    const request = {
+      domain: config.domain,
+      datasetId: config.datasetId,
+      format: config.format,
+      range: {
+        start: 0,
+        end: config.rangeEnd
+      }
+    }
+    const url = yield* buildExportUrl(request)
+
+    if (config.dryRun) {
+      return yield* writeSuccess(context, "socrata", {
+        dryRun: true,
+        method: "POST",
+        url: url.toString(),
+        headers: {
+          range: `bytes=0-${config.rangeEnd}`
+        }
+      })
+    }
+
+    return yield* runEffect(
+      context,
+      "socrata",
+      tokens,
+      Effect.gen(function* () {
+        const response = yield* exportResponse(request).pipe(
+          Effect.provide(context.runtime.soda3Layer)
+        )
+        const body = yield* Effect.promise(() => response.arrayBuffer())
+        return {
+          status: response.status,
+          byteLength: body.byteLength
+        }
+      })
+    )
+  })
+
+const socrataQueryCommand = Command.make(
+  "query",
+  {
+    domain: Flag.string("domain").pipe(
+      Flag.withDescription("Socrata domain without protocol, such as data.cityofnewyork.us.")
+    ),
+    datasetId: Flag.string("dataset").pipe(
+      Flag.withDescription("Socrata dataset id, such as ycrg-ses3.")
+    ),
+    select: Flag.string("select").pipe(
+      Flag.withDescription("SoQL select expression or full SELECT query.")
+    ),
+    dryRun: Flag.boolean("dry-run").pipe(
+      Flag.withDescription("Print the planned request without contacting the provider.")
+    )
+  },
+  handleSocrataQuery
+).pipe(Command.withShortDescription("Run a SODA3 query."))
+
+const socrataExportCommand = Command.make(
+  "export",
+  {
+    domain: Flag.string("domain").pipe(
+      Flag.withDescription("Socrata domain without protocol, such as data.cityofnewyork.us.")
+    ),
+    datasetId: Flag.string("dataset").pipe(
+      Flag.withDescription("Socrata dataset id, such as ycrg-ses3.")
+    ),
+    format: Flag.choice("format", ["csv", "json", "geojson"]).pipe(
+      Flag.withDefault("csv"),
+      Flag.withDescription("Export format. Defaults to csv.")
+    ),
+    output: Flag.string("output").pipe(
+      Flag.withDescription("Destination file path for the downloaded response body.")
+    ),
+    dryRun: Flag.boolean("dry-run").pipe(
+      Flag.withDescription("Print the planned request without contacting the provider.")
+    )
+  },
+  handleSocrataExport
+).pipe(Command.withShortDescription("Export a SODA3 dataset."))
+
+const socrataRangeProbeCommand = Command.make(
+  "range-probe",
+  {
+    domain: Flag.string("domain").pipe(
+      Flag.withDescription("Socrata domain without protocol, such as data.cityofnewyork.us.")
+    ),
+    datasetId: Flag.string("dataset").pipe(
+      Flag.withDescription("Socrata dataset id, such as ycrg-ses3.")
+    ),
+    format: Flag.choice("format", ["csv", "json", "geojson"]).pipe(
+      Flag.withDefault("csv"),
+      Flag.withDescription("Export format. Defaults to csv.")
+    ),
+    rangeEnd: Flag.integer("range-end").pipe(
+      Flag.withDescription("Inclusive byte offset for a bytes=0-N range probe.")
+    ),
+    dryRun: Flag.boolean("dry-run").pipe(
+      Flag.withDescription("Print the planned request without contacting the provider.")
+    )
+  },
+  handleSocrataRangeProbe
+).pipe(Command.withShortDescription("Probe SODA3 export range support."))
+
+export const socrataCommand = Command.make("socrata").pipe(
+  Command.withShortDescription("Socrata SODA3 commands."),
+  Command.withSubcommands([socrataQueryCommand, socrataExportCommand, socrataRangeProbeCommand])
+)

package/src/commands/soda3-shared.ts

@@ -0,0 +1,103 @@
+import type { Soda3ExportFormat } from "@nyc-transit-kit/contracts/soda3"
+import { buildExportUrl, buildQueryUrl } from "@nyc-transit-kit/soda3/endpoints"
+import * as Effect from "effect/Effect"
+import { atomicWrite } from "../files"
+import { runEffect } from "./shared"
+import type { ApiFamily, CommandContext } from "./types"
+
+type DryRunExtra = Readonly<Record<string, unknown>>
+
+export const fileWriteError = (cause: unknown) => ({
+  _tag: "FileWriteError",
+  message: cause instanceof Error ? cause.message : "File write failed."
+})
+
+const optionalHeader = (headers: Headers, name: string) => headers.get(name) ?? undefined
+
+const parseContentLength = (value: string | undefined) => {
+  if (value === undefined) {
+    return undefined
+  }
+
+  const parsed = Number(value)
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
+}
+
+export const writeSoda3QueryDryRun = (config: {
+  readonly context: CommandContext
+  readonly apiFamily: ApiFamily
+  readonly tokens: ReadonlyArray<string>
+  readonly domain: string
+  readonly datasetId: string
+  readonly query: string
+  readonly extra?: DryRunExtra
+}) =>
+  runEffect(
+    config.context,
+    config.apiFamily,
+    config.tokens,
+    buildQueryUrl({
+      domain: config.domain,
+      datasetId: config.datasetId,
+      query: config.query
+    }).pipe(
+      Effect.map((url) => ({
+        dryRun: true,
+        method: "POST",
+        url: url.toString(),
+        ...config.extra,
+        body: {
+          query: config.query
+        }
+      }))
+    )
+  )
+
+export const writeSoda3ExportDryRun = (config: {
+  readonly context: CommandContext
+  readonly apiFamily: ApiFamily
+  readonly tokens: ReadonlyArray<string>
+  readonly domain: string
+  readonly datasetId: string
+  readonly format: Soda3ExportFormat
+  readonly output: string
+  readonly extra?: DryRunExtra
+}) =>
+  runEffect(
+    config.context,
+    config.apiFamily,
+    config.tokens,
+    buildExportUrl({
+      domain: config.domain,
+      datasetId: config.datasetId,
+      format: config.format
+    }).pipe(
+      Effect.map((url) => ({
+        dryRun: true,
+        method: "POST",
+        url: url.toString(),
+        ...config.extra,
+        output: config.output
+      }))
+    )
+  )
+
+export const writeResponseToFile = (output: string, response: Response) =>
+  Effect.gen(function* () {
+    const contentType = optionalHeader(response.headers, "content-type")
+    const contentLength = parseContentLength(optionalHeader(response.headers, "content-length"))
+
+    yield* Effect.tryPromise({
+      try: () => atomicWrite(output, response),
+      catch: fileWriteError
+    })
+    const byteLength = yield* Effect.sync(() => Bun.file(output).size)
+
+    return {
+      output,
+      status: response.status,
+      byteLength,
+      ...(contentType === undefined ? {} : { contentType }),
+      ...(contentLength === undefined ? {} : { contentLength })
+    }
+  })

package/src/commands/types.ts

@@ -0,0 +1,33 @@
+import type { GtfsRealtimeDecoder } from "@nyc-transit-kit/mta/gtfs-realtime"
+import type { Soda3ClientConfig } from "@nyc-transit-kit/soda3/client"
+import * as Context from "effect/Context"
+import type * as Layer from "effect/Layer"
+import type * as HttpClient from "effect/unstable/http/HttpClient"
+import type { CliMetaApiFamily } from "../output"
+
+export type ApiFamily = CliMetaApiFamily
+
+export type CommandResult<A> =
+  | {
+      readonly _tag: "Failure"
+      readonly error: unknown
+    }
+  | {
+      readonly _tag: "Success"
+      readonly data: A
+    }
+
+export type CliRuntime = {
+  readonly soda3Layer: Layer.Layer<Soda3ClientConfig | HttpClient.HttpClient>
+  readonly mtaLayer: Layer.Layer<HttpClient.HttpClient>
+  readonly mtaRealtimeLayer: Layer.Layer<HttpClient.HttpClient | GtfsRealtimeDecoder>
+}
+
+export type CommandContext = {
+  readonly runtime: CliRuntime
+  readonly writeJson: (output: unknown) => void
+}
+
+export class CliCommandContext extends Context.Service<CliCommandContext, CommandContext>()(
+  "@nyc-transit-kit/cli/CliCommandContext"
+) {}

package/src/config.ts

@@ -0,0 +1,21 @@
+import { GtfsRealtimeDecoder } from "@nyc-transit-kit/mta/gtfs-realtime"
+import { MtaHttpLive } from "@nyc-transit-kit/mta/gtfs-static"
+import { Soda3ClientConfig, Soda3HttpLive } from "@nyc-transit-kit/soda3/client"
+import * as Layer from "effect/Layer"
+
+export interface Soda3CliLayerConfig {
+  readonly appToken?: string
+}
+
+export const soda3CliLayer = (config: Soda3CliLayerConfig) =>
+  Layer.mergeAll(
+    Layer.succeed(Soda3ClientConfig)({
+      appToken: config.appToken,
+      retryTimes: 2
+    }),
+    Soda3HttpLive
+  )
+
+export const mtaCliLayer = MtaHttpLive
+
+export const mtaRealtimeCliLayer = Layer.mergeAll(MtaHttpLive, GtfsRealtimeDecoder.Live)

package/src/files.ts

@@ -0,0 +1,27 @@
+import { rename } from "node:fs/promises"
+
+export type AtomicWriteBody =
+  | Blob
+  | NodeJS.TypedArray
+  | ArrayBufferLike
+  | string
+  | Bun.Archive
+  | Response
+
+export const atomicWrite = async (path: string, body: AtomicWriteBody) => {
+  const tempPath = `${path}.tmp-${process.pid}`
+
+  try {
+    if (body instanceof Response) {
+      await Bun.file(tempPath).write(body)
+    } else {
+      await Bun.write(tempPath, body)
+    }
+    await rename(tempPath, path)
+  } catch (cause) {
+    await Bun.file(tempPath)
+      .delete()
+      .catch(() => undefined)
+    throw cause
+  }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export const packageName = "@nyc-transit-kit/cli"
+export const localBinaryName = "ntk"

package/src/main.ts

@@ -0,0 +1,205 @@
+import * as BunServices from "@effect/platform-bun/BunServices"
+import { schemaVersion } from "@nyc-transit-kit/contracts/metadata"
+import * as Console from "effect/Console"
+import * as Effect from "effect/Effect"
+import * as CliError from "effect/unstable/cli/CliError"
+import * as Command from "effect/unstable/cli/Command"
+import { rootCommand } from "./commands/root"
+import { apiFamilyFor, commandLabel, writeJson } from "./commands/shared"
+import { CliCommandContext } from "./commands/types"
+import { localBinaryName } from "./index"
+import {
+  errorEnvelope,
+  packageBuildCommit,
+  packageBuildTarget,
+  packageVersion,
+  successEnvelope
+} from "./output"
+import { makeCliRuntime } from "./runtime"
+
+type CommandRunResult =
+  | {
+      readonly _tag: "Failure"
+      readonly error: unknown
+    }
+  | {
+      readonly _tag: "Success"
+    }
+
+type JsonErrorDetails = {
+  readonly code: string
+  readonly message: string
+  readonly tokens: ReadonlyArray<string>
+}
+
+const noop = () => undefined
+
+const quietConsole: Console.Console = {
+  assert: noop,
+  clear: noop,
+  count: noop,
+  countReset: noop,
+  debug: noop,
+  dir: noop,
+  dirxml: noop,
+  error: noop,
+  group: noop,
+  groupCollapsed: noop,
+  groupEnd: noop,
+  info: noop,
+  log: noop,
+  table: noop,
+  time: noop,
+  timeEnd: noop,
+  timeLog: noop,
+  trace: noop,
+  warn: noop
+}
+
+const stripRootCommand = (path: ReadonlyArray<string>) =>
+  path[0] === localBinaryName ? path.slice(1) : path
+
+const messageOf = (error: CliError.CliError) =>
+  "message" in error && typeof error.message === "string" ? error.message : "Command failed."
+
+const jsonErrorDetails = (
+  error: CliError.CliError,
+  fallbackPath: ReadonlyArray<string> = []
+): JsonErrorDetails => {
+  switch (error._tag) {
+    case "ShowHelp": {
+      const firstError = error.errors[0]
+      if (firstError !== undefined) {
+        return jsonErrorDetails(firstError, error.commandPath)
+      }
+
+      return {
+        code: "command-failed",
+        message: "Command failed.",
+        tokens: stripRootCommand(error.commandPath)
+      }
+    }
+    case "MissingOption":
+      return {
+        code: "missing-option",
+        message: error.message,
+        tokens: stripRootCommand(fallbackPath)
+      }
+    case "MissingArgument":
+      return {
+        code: "missing-argument",
+        message: error.message,
+        tokens: stripRootCommand(fallbackPath)
+      }
+    case "UnknownSubcommand": {
+      const parent = error.parent ?? fallbackPath
+      return {
+        code: "unknown-command",
+        message: error.message,
+        tokens: stripRootCommand([...parent, error.subcommand])
+      }
+    }
+    case "UnrecognizedOption":
+      return {
+        code: "unknown-option",
+        message: error.message,
+        tokens: stripRootCommand(error.command ?? fallbackPath)
+      }
+    case "InvalidValue":
+      return {
+        code: "invalid-option",
+        message: error.message,
+        tokens: stripRootCommand(fallbackPath)
+      }
+    case "DuplicateOption":
+    case "UserError":
+      return {
+        code: "command-failed",
+        message: messageOf(error),
+        tokens: stripRootCommand(fallbackPath)
+      }
+  }
+}
+
+const writeJsonFailure = (error: unknown) => {
+  const details = CliError.isCliError(error)
+    ? jsonErrorDetails(error)
+    : {
+        code: "command-failed",
+        message: "Command failed.",
+        tokens: []
+      }
+
+  process.exitCode = 1
+  writeJson(
+    errorEnvelope(apiFamilyFor(details.tokens), {
+      code: details.code,
+      message: details.message,
+      command: commandLabel(details.tokens)
+    })
+  )
+}
+
+const versionJson = () =>
+  successEnvelope("cli", {
+    binary: localBinaryName,
+    version: packageVersion,
+    schemaVersion,
+    commit: packageBuildCommit,
+    buildTarget: packageBuildTarget
+  })
+
+export const main = async () => {
+  const args = Bun.argv.slice(2)
+  if (args.includes("--version")) {
+    if (args.includes("--json")) {
+      writeJson(versionJson())
+      return
+    }
+
+    console.log(`${localBinaryName} ${packageVersion}`)
+    return
+  }
+
+  const context = {
+    runtime: makeCliRuntime(process.env),
+    writeJson
+  }
+  const baseProgram = Command.runWith(rootCommand, { version: packageVersion })(args).pipe(
+    Effect.provideService(CliCommandContext, context),
+    Effect.provide(BunServices.layer)
+  )
+  const shouldKeepJsonClean = args.includes("--json") && !args.includes("--help")
+  const program = shouldKeepJsonClean
+    ? baseProgram.pipe(Effect.provideService(Console.Console, quietConsole))
+    : baseProgram
+
+  const result = await Effect.runPromise(
+    program.pipe(
+      Effect.match({
+        onFailure: (error): CommandRunResult => ({
+          _tag: "Failure",
+          error
+        }),
+        onSuccess: (): CommandRunResult => ({
+          _tag: "Success"
+        })
+      })
+    )
+  )
+
+  if (result._tag === "Success") {
+    return
+  }
+
+  if (args.includes("--json")) {
+    writeJsonFailure(result.error)
+    return
+  }
+
+  process.exitCode = 1
+}
+
+if (import.meta.main) {
+  await main()
+}

package/src/output.ts

@@ -0,0 +1,39 @@
+import { releaseVersion, schemaVersion } from "@nyc-transit-kit/contracts/metadata"
+import { buildCommit, buildTarget } from "./build-info"
+
+export const packageVersion = releaseVersion
+export const packageBuildCommit = buildCommit
+export const packageBuildTarget = buildTarget
+
+const currentGeneratedAt = () => new Date().toISOString()
+
+export type CliMetaApiFamily = "socrata" | "mta" | "nyc-open-data" | "nyc-dot" | "cli"
+
+export const successEnvelope = (apiFamily: CliMetaApiFamily, data: unknown) => ({
+  ok: true,
+  data,
+  meta: {
+    apiFamily,
+    generatedAt: currentGeneratedAt(),
+    schemaVersion
+  }
+})
+
+export const errorEnvelope = (
+  apiFamily: CliMetaApiFamily,
+  error: {
+    readonly code: string
+    readonly message: string
+    readonly provider?: string
+    readonly retryable?: boolean
+    readonly command?: string
+  }
+) => ({
+  ok: false,
+  error,
+  meta: {
+    apiFamily,
+    generatedAt: currentGeneratedAt(),
+    schemaVersion
+  }
+})

package/src/runtime.ts

@@ -0,0 +1,14 @@
+import type { CliRuntime } from "./commands/types"
+import { mtaCliLayer, mtaRealtimeCliLayer, soda3CliLayer } from "./config"
+
+export const makeCliRuntime = (env: Readonly<Record<string, string | undefined>>): CliRuntime => {
+  const soda3Layer = soda3CliLayer({
+    appToken: env.SOCRATA_APP_TOKEN
+  })
+
+  return {
+    soda3Layer,
+    mtaLayer: mtaCliLayer,
+    mtaRealtimeLayer: mtaRealtimeCliLayer
+  }
+}