@nyc-transit-kit/soda3 0.1.4 → 0.3.0

package/src/Soql.ts

@@ -0,0 +1,200 @@
+/**
+ * SoQL as `effect/sql` fragments.
+ *
+ * Text written in a `soql` template is trusted SoQL. Every interpolated value
+ * becomes a quoted literal and every `identifier` a backticked column, so input
+ * can never change the shape of a query. SODA3 has no bind parameters (`:name`
+ * names a system column), so `compile` writes the values into the text.
+ */
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import * as Statement from "effect/sql/Statement"
+import type { IsoDate } from "./Socrata"
+
+export type Fragment = Statement.Fragment
+
+/** A value SoQL can write as a literal. Timestamps are strings such as `2026-10-01T00:00:00`. */
+export type Value = string | number | boolean
+
+/** A fragment that cannot be written as SoQL, or clauses that make no sense. */
+export class SoqlError extends Schema.TaggedError<SoqlError>()("SoqlError", {
+  message: Schema.String
+}) {}
+
+/** A column or alias, written as `` `name` `` so that reserved words such as `from` work. */
+export const identifier = (name: string): Statement.Identifier => Statement.identifier(name)
+
+/**
+ * `` soql`speed > ${limit}` ``. The text is SoQL; each interpolation is a value,
+ * an `identifier`, or another fragment spliced in place.
+ */
+export const soql = (
+  strings: TemplateStringsArray,
+  ...args: ReadonlyArray<Value | Fragment | Statement.Segment>
+): Fragment => {
+  const segments: Array<Statement.Segment> = []
+  strings.forEach((text, index) => {
+    if (text !== "") {
+      segments.push(Statement.literal(text))
+    }
+    if (index < args.length) {
+      const arg = args[index]
+      if (Statement.isFragment(arg)) {
+        segments.push(...arg.segments)
+      } else if (typeof arg === "object") {
+        segments.push(arg)
+      } else if (arg !== undefined) {
+        segments.push(Statement.parameter(arg))
+      }
+    }
+  })
+  return Statement.fragment(segments)
+}
+
+const quote = (text: string) => `'${text.replaceAll("'", "''")}'`
+
+const write = (value: unknown) => (typeof value === "string" ? quote(value) : String(value))
+
+const compiler = Statement.makeCompiler({
+  // `Statement` knows no SoQL dialect. The name only matters to dialect switches, which SoQL never uses.
+  dialect: "pg",
+  placeholder: (_index, value) => write(value),
+  onIdentifier: (name) => `\`${name}\``,
+  onRecordUpdate: () => ["", []],
+  onCustom: () => ["", []]
+})
+
+const invalidValue = (value: unknown) =>
+  typeof value === "string" ||
+  typeof value === "boolean" ||
+  (typeof value === "number" && Number.isFinite(value))
+    ? undefined
+    : `SoQL has no literal for ${String(value)}`
+
+const invalidSegment = (segment: Statement.Segment): string | undefined => {
+  switch (segment._tag) {
+    case "Literal":
+      return undefined
+    case "Identifier":
+      // SoQL has no escape for a backtick inside a quoted name.
+      return segment.value === "" || segment.value.includes("`")
+        ? `SoQL cannot name the column ${JSON.stringify(segment.value)}`
+        : undefined
+    case "Parameter":
+      return invalidValue(segment.value)
+    case "ArrayHelper":
+      return segment.value.length === 0
+        ? "SoQL IN needs at least one value"
+        : segment.value.map(invalidValue).find((problem) => problem !== undefined)
+    default:
+      return `SoQL has no ${segment._tag}`
+  }
+}
+
+/** The SoQL text of a fragment. Fails on a column SoQL cannot name or a value it cannot write. */
+export const compile = (fragment: Fragment): Effect.Effect<string, SoqlError> => {
+  const problem = fragment.segments.map(invalidSegment).find((message) => message !== undefined)
+  return problem === undefined
+    ? Effect.succeed(compiler.compile(fragment, false)[0])
+    : Effect.fail(new SoqlError({ message: problem }))
+}
+
+export const eq = (column: string, value: Value): Fragment => soql`${identifier(column)} = ${value}`
+
+export const isIn = (column: string, values: readonly [Value, ...Array<Value>]): Fragment =>
+  soql`${identifier(column)} IN ${Statement.arrayHelper(values)}`
+
+export const isNotNull = (column: string): Fragment => soql`${identifier(column)} IS NOT NULL`
+
+const joinParenthesized =
+  (separator: string) =>
+  (clauses: readonly [Fragment, ...Array<Fragment>]): Fragment =>
+    Statement.join(separator, false)(clauses.map((clause) => soql`(${clause})`))
+
+/** `(a) AND (b)`. Each clause keeps its own parentheses, so an `OR` inside one stays inside it. */
+export const and = joinParenthesized(" AND ")
+
+/** `(a) OR (b)`. */
+export const or = joinParenthesized(" OR ")
+
+export type SortDirection = "ASC" | "DESC"
+
+export interface OrderBy {
+  readonly column: string
+  readonly direction?: SortDirection | undefined
+}
+
+/** The clauses of a query over whole rows. */
+export interface Clauses {
+  readonly where?: Fragment | undefined
+  readonly orderBy?: OrderBy | undefined
+  readonly limit?: number | undefined
+}
+
+/** `SELECT *` with each clause that is given, in SoQL's order. */
+export const selectAll = ({ where, orderBy, limit }: Clauses): Fragment =>
+  Statement.fragment([
+    Statement.literal("SELECT *"),
+    ...(where === undefined ? [] : [Statement.literal(" WHERE "), ...where.segments]),
+    ...(orderBy === undefined
+      ? []
+      : [
+          Statement.literal(" ORDER BY "),
+          identifier(orderBy.column),
+          Statement.literal(` ${orderBy.direction ?? "ASC"}`)
+        ]),
+    ...(limit === undefined ? [] : [Statement.literal(" LIMIT "), Statement.parameter(limit)])
+  ])
+
+/** A half-open window of Socrata floating timestamps, `start <= t < end`. */
+export interface TimestampWindow {
+  readonly start: string
+  readonly end: string
+}
+
+export interface YearMonth {
+  readonly year: number
+  readonly month: 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12
+}
+
+const monthStart = (year: number, month: number) =>
+  `${year.toString().padStart(4, "0")}-${month.toString().padStart(2, "0")}-01T00:00:00`
+
+/** Midnight at the start of a calendar date, as a floating timestamp. */
+export const timestamp = (date: IsoDate) => `${date}T00:00:00`
+
+/** The calendar month as a window. */
+export const monthWindow = ({ year, month }: YearMonth): TimestampWindow => ({
+  start: monthStart(year, month),
+  end: month === 12 ? monthStart(year + 1, 1) : monthStart(year, month + 1)
+})
+
+/** The days from `start` up to, not including, `end`. Fails when the window is empty. */
+export const dateWindow = (start: IsoDate, end: IsoDate) =>
+  start < end
+    ? Effect.succeed<TimestampWindow>({ start: timestamp(start), end: timestamp(end) })
+    : Effect.fail(new SoqlError({ message: `Date window ${start}..${end} is empty` }))
+
+/** `column` inside the window. */
+export const timestampRange = (column: string, window: TimestampWindow): Fragment =>
+  soql`${identifier(column)} >= ${window.start} AND ${identifier(column)} < ${window.end}`
+
+/** `column` inside the calendar month. */
+export const monthRange = (column: string, month: YearMonth): Fragment =>
+  timestampRange(column, monthWindow(month))
+
+/** Rows whose separate year and month columns fall in `[start, end]`, both months included. */
+export const yearMonthRange = (
+  columns: { readonly year: string; readonly month: string },
+  start: YearMonth,
+  end: YearMonth
+) => {
+  if (start.year * 12 + start.month > end.year * 12 + end.month) {
+    return Effect.fail(new SoqlError({ message: "Year/month range starts after it ends" }))
+  }
+  const year = identifier(columns.year)
+  const month = identifier(columns.month)
+  return Effect.succeed(
+    soql`(${year} > ${start.year} OR (${year} = ${start.year} AND ${month} >= ${start.month})) AND (${year} < ${end.year} OR (${year} = ${end.year} AND ${month} <= ${end.month}))`
+  )
+}

package/src/index.ts

@@ -1,61 +1,5 @@
-export const packageName = "@nyc-transit-kit/soda3"
-export { buildCatalogSearchUrl, catalogSearch } from "./catalog"
-export type { FetchImplementation, Soda3ClientConfigShape, Soda3LayerOptions } from "./client"
-export {
-  Soda3ClientConfig,
-  Soda3HttpLive,
-  Soda3Live,
-  soda3ConfigLayer,
-  soda3FetchLayer,
-  soda3Layer
-} from "./client"
-export {
-  buildExportUrl,
-  buildQueryUrl,
-  decodeSoda3CatalogSearchRequest,
-  decodeSoda3ExportRequest,
-  decodeSoda3QueryRequest,
-  defaultSocrataProtocol,
-  discoveryApiHost,
-  socrataApiVersion
-} from "./endpoints"
-export type { Soda3ClientError } from "./errors"
-export {
-  InvalidInputError,
-  isRetryableProviderError,
-  isSoda3ClientError,
-  ProviderContractError,
-  ProviderHttpError,
-  RetryExhaustedError,
-  TimeoutError
-} from "./errors"
-export { exportResponse } from "./export"
-export type { Soda3DatasetMetadataRequestInput } from "./metadata"
-export { fetchDatasetMetadata } from "./metadata"
-export type { Soda3CountRowsInput, Soda3QueryAllRowsInput } from "./query"
-export { countRows, queryAllRows, queryRows } from "./query"
-export type {
-  SocrataTimestampWindow,
-  SocrataYearMonth,
-  SoqlFragment,
-  SoqlParameters,
-  SoqlSortDirection
-} from "./soql"
-export {
-  socrataDateWindow,
-  socrataMonthWindow,
-  socrataTimestamp,
-  soqlAnd,
-  soqlEq,
-  soqlIdentifier,
-  soqlIn,
-  soqlIsNotNull,
-  soqlLimit,
-  soqlMonthWindow,
-  soqlOrderBy,
-  soqlParameter,
-  soqlParameterName,
-  soqlSelectAll,
-  soqlTimestampRange,
-  soqlYearMonthRange
-} from "./soql"
+export * as Dataset from "./Dataset"
+export * as Discovery from "./Discovery"
+export * as Socrata from "./Socrata"
+export * as Soda3Client from "./Soda3Client"
+export * as Soql from "./Soql"

package/src/catalog.ts

@@ -1,28 +0,0 @@
-import type { Soda3CatalogSearchRequestInput } from "@nyc-transit-kit/contracts/soda3"
-import * as Effect from "effect/Effect"
-import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
-import { buildCatalogSearchUrl } from "./endpoints"
-import { decodeCatalogSearchResponse, decodeJson } from "./internal/responses"
-import { Soda3ClientConfig } from "./internal/services"
-import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transport"
-
-export { buildCatalogSearchUrl } from "./endpoints"
-export type { Soda3CatalogSearchRequestInput }
-
-export const catalogSearch = Effect.fn("Soda3.catalogSearch")(function* (
-  input: Soda3CatalogSearchRequestInput
-) {
-  const config = yield* Soda3ClientConfig
-  const url = yield* buildCatalogSearchUrl(input)
-  const request = HttpClientRequest.get(url, {
-    headers: addHeaders(config)
-  })
-
-  return yield* requestWithPolicy(
-    "catalog",
-    executeRequest("catalog", request).pipe(
-      Effect.flatMap((response) => decodeJson("catalog", response)),
-      Effect.flatMap(decodeCatalogSearchResponse)
-    )
-  )
-})

package/src/client.ts

@@ -1,17 +0,0 @@
-export { catalogSearch } from "./catalog"
-export { exportResponse } from "./export"
-export { Soda3HttpLive } from "./internal/http"
-export type {
-  FetchImplementation,
-  Soda3ClientConfigShape,
-  Soda3LayerOptions
-} from "./internal/services"
-export {
-  Soda3ClientConfig,
-  Soda3Live,
-  soda3ConfigLayer,
-  soda3FetchLayer,
-  soda3Layer
-} from "./internal/services"
-export { fetchDatasetMetadata } from "./metadata"
-export { countRows, queryAllRows, queryRows } from "./query"

package/src/endpoints.ts

@@ -1,110 +0,0 @@
-import {
-  Soda3CatalogSearchRequest,
-  type Soda3CatalogSearchRequestInput,
-  Soda3ExportRequest,
-  type Soda3ExportRequestInput,
-  Soda3QueryRequest,
-  type Soda3QueryRequestInput
-} from "@nyc-transit-kit/contracts/soda3"
-import * as Effect from "effect/Effect"
-import * as Schema from "effect/Schema"
-import { InvalidInputError } from "./errors"
-
-export const socrataApiVersion = "v3"
-export const defaultSocrataProtocol = "https"
-export const discoveryApiHost = "api.us.socrata.com"
-
-const decodeQueryRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3QueryRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "query",
-          message: error.message
-        })
-      )
-    )
-  )
-
-const decodeExportRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3ExportRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "export",
-          message: error.message
-        })
-      )
-    )
-  )
-
-const decodeCatalogSearchRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3CatalogSearchRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "catalog",
-          message: error.message
-        })
-      )
-    )
-  )
-
-const viewUrl = (domain: string, datasetId: string, suffix: string) => {
-  const url = new URL(`${defaultSocrataProtocol}://${domain}`)
-  url.pathname = `/api/${socrataApiVersion}/views/${datasetId}/${suffix}`
-  return url
-}
-
-export const buildQueryUrl = Effect.fn("Soda3.buildQueryUrl")(function* (
-  input: Soda3QueryRequestInput
-) {
-  const request = yield* decodeQueryRequest(input)
-  return viewUrl(request.domain, request.datasetId, "query.json")
-})
-
-export const buildExportUrl = Effect.fn("Soda3.buildExportUrl")(function* (
-  input: Soda3ExportRequestInput
-) {
-  const request = yield* decodeExportRequest(input)
-  return viewUrl(request.domain, request.datasetId, `export.${request.format}`)
-})
-
-export const buildCatalogSearchUrl = Effect.fn("Soda3.buildCatalogSearchUrl")(function* (
-  input: Soda3CatalogSearchRequestInput
-) {
-  const request = yield* decodeCatalogSearchRequest(input)
-  const url = new URL(`${defaultSocrataProtocol}://${discoveryApiHost}`)
-  url.pathname = "/api/catalog/v1"
-  url.searchParams.set("domains", request.domain)
-  url.searchParams.set("search_context", request.domain)
-  url.searchParams.set("only", "dataset")
-
-  if (request.query !== undefined) {
-    url.searchParams.set("search", request.query)
-  }
-  for (const id of request.ids ?? []) {
-    url.searchParams.append("ids", id)
-  }
-  for (const category of request.categories ?? []) {
-    url.searchParams.append("categories", category)
-  }
-  for (const tag of request.tags ?? []) {
-    url.searchParams.append("tags", tag)
-  }
-  if (request.order !== undefined) {
-    url.searchParams.set("order", request.order)
-  }
-  if (request.limit !== undefined) {
-    url.searchParams.set("limit", request.limit.toString())
-  }
-  if (request.offset !== undefined) {
-    url.searchParams.set("offset", request.offset.toString())
-  }
-
-  return url
-})
-
-export const decodeSoda3QueryRequest = decodeQueryRequest
-export const decodeSoda3ExportRequest = decodeExportRequest
-export const decodeSoda3CatalogSearchRequest = decodeCatalogSearchRequest

package/src/errors.ts

@@ -1,58 +0,0 @@
-import * as Schema from "effect/Schema"
-
-export class InvalidInputError extends Schema.TaggedErrorClass<InvalidInputError>()(
-  "InvalidInputError",
-  {
-    operation: Schema.String,
-    message: Schema.String
-  }
-) {}
-
-export class ProviderHttpError extends Schema.TaggedErrorClass<ProviderHttpError>()(
-  "ProviderHttpError",
-  {
-    operation: Schema.String,
-    status: Schema.Number,
-    statusText: Schema.String,
-    retryable: Schema.Boolean
-  }
-) {}
-
-export class ProviderContractError extends Schema.TaggedErrorClass<ProviderContractError>()(
-  "ProviderContractError",
-  {
-    operation: Schema.String,
-    message: Schema.String
-  }
-) {}
-
-export class TimeoutError extends Schema.TaggedErrorClass<TimeoutError>()("TimeoutError", {
-  operation: Schema.String,
-  timeoutMs: Schema.Number
-}) {}
-
-export class RetryExhaustedError extends Schema.TaggedErrorClass<RetryExhaustedError>()(
-  "RetryExhaustedError",
-  {
-    operation: Schema.String,
-    attempts: Schema.Number,
-    message: Schema.String
-  }
-) {}
-
-export type Soda3ClientError =
-  | InvalidInputError
-  | ProviderHttpError
-  | ProviderContractError
-  | TimeoutError
-  | RetryExhaustedError
-
-export const isSoda3ClientError = (value: unknown): value is Soda3ClientError =>
-  value instanceof InvalidInputError ||
-  value instanceof ProviderHttpError ||
-  value instanceof ProviderContractError ||
-  value instanceof TimeoutError ||
-  value instanceof RetryExhaustedError
-
-export const isRetryableProviderError = (error: Soda3ClientError): error is ProviderHttpError =>
-  error._tag === "ProviderHttpError" && error.retryable

package/src/export.ts

@@ -1,41 +0,0 @@
-import type { Soda3ExportRequestInput } from "@nyc-transit-kit/contracts/soda3"
-import * as Effect from "effect/Effect"
-import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
-import { buildExportUrl, decodeSoda3ExportRequest } from "./endpoints"
-import { InvalidInputError } from "./errors"
-import { rangeHeader, toExportBody } from "./internal/bodies"
-import { toWebResponse } from "./internal/http"
-import { Soda3ClientConfig } from "./internal/services"
-import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transport"
-
-export type { Soda3ExportRequestInput }
-
-export const exportResponse = Effect.fn("Soda3.exportResponse")(function* (
-  input: Soda3ExportRequestInput
-) {
-  const config = yield* Soda3ClientConfig
-  const exportRequest = yield* decodeSoda3ExportRequest(input)
-  const url = yield* buildExportUrl(exportRequest)
-  const headers = addHeaders(config)
-  const range = rangeHeader(exportRequest)
-
-  const request = yield* HttpClientRequest.post(url, {
-    headers: {
-      ...headers,
-      range
-    }
-  }).pipe(
-    HttpClientRequest.bodyJson(toExportBody(exportRequest)),
-    Effect.mapError(() =>
-      InvalidInputError.make({
-        operation: "export",
-        message: "SODA3 export body was not JSON serializable"
-      })
-    )
-  )
-
-  return yield* requestWithPolicy(
-    "export",
-    executeRequest("export", request).pipe(Effect.flatMap(toWebResponse))
-  )
-})

package/src/internal/bodies.ts

@@ -1,23 +0,0 @@
-import type { Soda3ExportRequest, Soda3QueryRequest } from "@nyc-transit-kit/contracts/soda3"
-
-export const toQueryBody = (request: Soda3QueryRequest) => ({
-  query: request.query,
-  page: request.page,
-  parameters: request.parameters,
-  timeout: request.timeout,
-  includeSystem: request.includeSystem,
-  includeSynthetic: request.includeSynthetic,
-  orderingSpecifier: request.orderingSpecifier
-})
-
-export const toExportBody = (request: Soda3ExportRequest) => ({
-  query: request.query,
-  parameters: request.parameters,
-  timeout: request.timeout,
-  serializationOptions: request.serializationOptions
-})
-
-export const rangeHeader = (request: Soda3ExportRequest) =>
-  request.range === undefined
-    ? undefined
-    : `bytes=${request.range.start}-${request.range.end ?? ""}`

package/src/internal/http.ts

@@ -1,42 +0,0 @@
-import * as Effect from "effect/Effect"
-import * as Stream from "effect/Stream"
-import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
-import type * as HttpClientError from "effect/unstable/http/HttpClientError"
-import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
-import { ProviderHttpError } from "../errors"
-
-export const Soda3HttpLive = FetchHttpClient.layer
-
-export const retryableStatus = (status: number) => status === 429 || status >= 500
-
-const httpStatusText = (status: number) => `HTTP ${status}`
-
-export const toWebResponse = (response: HttpClientResponse.HttpClientResponse) =>
-  Effect.map(
-    Stream.toReadableStreamEffect(response.stream),
-    (body) =>
-      new Response(body, {
-        status: response.status,
-        headers: response.headers
-      })
-  )
-
-export const mapHttpClientError = (operation: string, error: HttpClientError.HttpClientError) => {
-  const response = error.response
-
-  if (response !== undefined) {
-    return ProviderHttpError.make({
-      operation,
-      status: response.status,
-      statusText: httpStatusText(response.status),
-      retryable: retryableStatus(response.status)
-    })
-  }
-
-  return ProviderHttpError.make({
-    operation,
-    status: 0,
-    statusText: error.message,
-    retryable: true
-  })
-}

package/src/internal/responses.ts

@@ -1,56 +0,0 @@
-import {
-  Soda3CatalogSearchResponse,
-  Soda3QueryResponse,
-  Soda3Row
-} from "@nyc-transit-kit/contracts/soda3"
-import * as Effect from "effect/Effect"
-import * as Schema from "effect/Schema"
-import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
-import { ProviderContractError } from "../errors"
-
-const providerRows = Schema.Array(Soda3Row)
-
-export const decodeJson = (operation: string, response: HttpClientResponse.HttpClientResponse) =>
-  response.json.pipe(
-    Effect.catchTag("HttpClientError", () =>
-      Effect.fail(
-        ProviderContractError.make({
-          operation,
-          message: "Provider response was not valid JSON"
-        })
-      )
-    )
-  )
-
-export const decodeQueryResponse = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3QueryResponse)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Schema.decodeUnknownEffect(providerRows)(input).pipe(
-        Effect.map((rows) =>
-          Soda3QueryResponse.make({
-            rows
-          })
-        ),
-        Effect.catchTag("SchemaError", () =>
-          Effect.fail(
-            ProviderContractError.make({
-              operation: "query",
-              message: error.message
-            })
-          )
-        )
-      )
-    )
-  )
-
-export const decodeCatalogSearchResponse = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3CatalogSearchResponse)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        ProviderContractError.make({
-          operation: "catalog",
-          message: error.message
-        })
-      )
-    )
-  )