@nyc-transit-kit/soda3 0.1.3 → 0.2.0

package/README.md

@@ -1,5 +1,30 @@
 # @nyc-transit-kit/soda3
 
-Part of `nyc-transit-kit`, an Effect-native Bun toolkit for official NYC and MTA transit data APIs.
+Generic Effect-native client for Socrata SODA3 query, export, and Discovery
+catalog APIs.
 
-See the repository README and docs/api-reference.md for public import paths, CLI commands, and release notes.
+Socrata-backed data access is SODA3-only. This package does not expose legacy
+Socrata endpoint helpers, fallback endpoint builders, or compatibility aliases.
+
+## Import Paths
+
+```ts
+import { soda3Layer } from "@nyc-transit-kit/soda3/client"
+import { queryRows } from "@nyc-transit-kit/soda3/query"
+import * as Effect from "effect/Effect"
+
+const program = queryRows({
+  domain: "data.cityofnewyork.us",
+  datasetId: "ycrg-ses3",
+  query: "SELECT * LIMIT 5"
+}).pipe(Effect.provide(soda3Layer({ retryTimes: 2 })))
+```
+
+Prefer subpaths such as `@nyc-transit-kit/soda3/query`,
+`@nyc-transit-kit/soda3/export`, `@nyc-transit-kit/soda3/catalog`, and
+`@nyc-transit-kit/soda3/client`.
+
+## Compatibility
+
+Each `nyc-transit-kit` release pins one exact Effect 4 beta. Match the Effect
+version documented in `docs/getting-started.md`.

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@nyc-transit-kit/soda3",
-  "version": "0.1.3",
+  "version": "0.2.0",
+  "license": "MIT",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -10,8 +11,10 @@
     "./endpoints": "./src/endpoints.ts",
     "./errors": "./src/errors.ts",
     "./export": "./src/export.ts",
+    "./metadata": "./src/metadata.ts",
     "./query": "./src/query.ts",
-    "./soql": "./src/soql.ts"
+    "./soql": "./src/soql.ts",
+    "./transport": "./src/transport.ts"
   },
   "types": "./src/index.ts",
   "files": [
@@ -24,13 +27,12 @@
     "test": "bun test ./test"
   },
   "dependencies": {
-    "@nyc-transit-kit/contracts": "0.1.3",
+    "@nyc-transit-kit/contracts": "0.2.0",
     "effect": "4.0.0-beta.92"
   },
   "publishConfig": {
     "access": "public"
   },
-  "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mannyc2/nyc-transit-kit.git",

package/src/catalog.ts

@@ -1,10 +1,15 @@
 import type { Soda3CatalogSearchRequestInput } from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
+import { Soda3ClientConfig } from "./client"
 import { buildCatalogSearchUrl } from "./endpoints"
-import { decodeCatalogSearchResponse, decodeJson } from "./internal/responses"
-import { Soda3ClientConfig } from "./internal/services"
-import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transport"
+import {
+  addHeaders,
+  decodeCatalogSearchResponse,
+  decodeJson,
+  executeRequest,
+  requestWithPolicy
+} from "./internal/transport"
 
 export { buildCatalogSearchUrl } from "./endpoints"
 export type { Soda3CatalogSearchRequestInput }

package/src/client.ts

@@ -1,12 +1,44 @@
+import * as Context from "effect/Context"
+import * as Layer from "effect/Layer"
+import { type FetchImplementation, fetchHttpClientLayer } from "./transport"
+
 export { catalogSearch } from "./catalog"
 export { exportResponse } from "./export"
-export { Soda3HttpLive } from "./internal/http"
-export type { Soda3ClientConfigShape, Soda3LayerOptions } from "./internal/services"
-export {
-  Soda3ClientConfig,
-  Soda3Live,
-  soda3ConfigLayer,
-  soda3FetchLayer,
-  soda3Layer
-} from "./internal/services"
-export { queryRows } from "./query"
+export { fetchDatasetMetadata } from "./metadata"
+export { countRows, queryAllRows, queryRows } from "./query"
+export type { FetchImplementation } from "./transport"
+
+export interface Soda3ClientConfigShape {
+  readonly appToken?: string
+  readonly retryTimes?: number
+  readonly retryDelayMs?: number
+  readonly timeoutMs?: number
+}
+
+export interface Soda3LayerOptions extends Soda3ClientConfigShape {
+  readonly fetch?: FetchImplementation
+}
+
+export class Soda3ClientConfig extends Context.Service<Soda3ClientConfig, Soda3ClientConfigShape>()(
+  "@nyc-transit-kit/soda3/Soda3ClientConfig"
+) {
+  static readonly Default = Layer.succeed(Soda3ClientConfig)({
+    retryTimes: 0
+  })
+}
+
+export const soda3ConfigLayer = (options: Soda3ClientConfigShape = {}) =>
+  Layer.succeed(Soda3ClientConfig)({
+    retryTimes: options.retryTimes ?? 0,
+    ...(options.appToken === undefined ? {} : { appToken: options.appToken }),
+    ...(options.retryDelayMs === undefined ? {} : { retryDelayMs: options.retryDelayMs }),
+    ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs })
+  })
+
+export const Soda3HttpLive = fetchHttpClientLayer()
+export const soda3FetchLayer = fetchHttpClientLayer
+
+export const soda3Layer = (options: Soda3LayerOptions = {}) =>
+  Layer.mergeAll(soda3ConfigLayer(options), soda3FetchLayer(options.fetch))
+
+export const Soda3Live = soda3Layer()

package/src/endpoints.ts

@@ -7,48 +7,15 @@ import {
   type Soda3QueryRequestInput
 } from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
-import * as Schema from "effect/Schema"
-import { InvalidInputError } from "./errors"
+import { decodeInput } from "./internal/decode"
 
 export const socrataApiVersion = "v3"
 export const defaultSocrataProtocol = "https"
 export const discoveryApiHost = "api.us.socrata.com"
 
-const decodeQueryRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3QueryRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "query",
-          message: error.message
-        })
-      )
-    )
-  )
-
-const decodeExportRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3ExportRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "export",
-          message: error.message
-        })
-      )
-    )
-  )
-
-const decodeCatalogSearchRequest = (input: unknown) =>
-  Schema.decodeUnknownEffect(Soda3CatalogSearchRequest)(input).pipe(
-    Effect.catchTag("SchemaError", (error) =>
-      Effect.fail(
-        InvalidInputError.make({
-          operation: "catalog",
-          message: error.message
-        })
-      )
-    )
-  )
+const decodeQueryRequest = decodeInput("query", Soda3QueryRequest)
+const decodeExportRequest = decodeInput("export", Soda3ExportRequest)
+const decodeCatalogSearchRequest = decodeInput("catalog", Soda3CatalogSearchRequest)
 
 const viewUrl = (domain: string, datasetId: string, suffix: string) => {
   const url = new URL(`${defaultSocrataProtocol}://${domain}`)
@@ -83,6 +50,18 @@ export const buildCatalogSearchUrl = Effect.fn("Soda3.buildCatalogSearchUrl")(fu
   if (request.query !== undefined) {
     url.searchParams.set("search", request.query)
   }
+  for (const id of request.ids ?? []) {
+    url.searchParams.append("ids", id)
+  }
+  for (const category of request.categories ?? []) {
+    url.searchParams.append("categories", category)
+  }
+  for (const tag of request.tags ?? []) {
+    url.searchParams.append("tags", tag)
+  }
+  if (request.order !== undefined) {
+    url.searchParams.set("order", request.order)
+  }
   if (request.limit !== undefined) {
     url.searchParams.set("limit", request.limit.toString())
   }

package/src/export.ts

@@ -1,12 +1,17 @@
 import type { Soda3ExportRequestInput } from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
+import { Soda3ClientConfig } from "./client"
 import { buildExportUrl, decodeSoda3ExportRequest } from "./endpoints"
 import { InvalidInputError } from "./errors"
-import { rangeHeader, toExportBody } from "./internal/bodies"
-import { toWebResponse } from "./internal/http"
-import { Soda3ClientConfig } from "./internal/services"
-import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transport"
+import {
+  addHeaders,
+  executeRequest,
+  rangeHeader,
+  requestWithPolicy,
+  toExportBody,
+  toWebResponse
+} from "./internal/transport"
 
 export type { Soda3ExportRequestInput }
 

package/src/index.ts

@@ -1,6 +1,6 @@
 export const packageName = "@nyc-transit-kit/soda3"
 export { buildCatalogSearchUrl, catalogSearch } from "./catalog"
-export type { Soda3ClientConfigShape, Soda3LayerOptions } from "./client"
+export type { FetchImplementation, Soda3ClientConfigShape, Soda3LayerOptions } from "./client"
 export {
   Soda3ClientConfig,
   Soda3HttpLive,
@@ -30,7 +30,10 @@ export {
   TimeoutError
 } from "./errors"
 export { exportResponse } from "./export"
-export { queryRows } from "./query"
+export type { Soda3DatasetMetadataRequestInput } from "./metadata"
+export { fetchDatasetMetadata } from "./metadata"
+export type { Soda3CountRowsInput, Soda3QueryAllRowsInput } from "./query"
+export { countRows, queryAllRows, queryRows } from "./query"
 export type {
   SocrataTimestampWindow,
   SocrataYearMonth,
@@ -56,3 +59,15 @@ export {
   soqlTimestampRange,
   soqlYearMonthRange
 } from "./soql"
+export type { RequestPolicyConfig } from "./transport"
+export {
+  contentLengthOf,
+  fetchHttpClientLayer,
+  httpFailureInfo,
+  httpStatusText,
+  isOkStatus,
+  redactedUrl,
+  retryableStatus,
+  toWebResponse,
+  withRequestPolicy
+} from "./transport"

package/src/internal/decode.ts

@@ -0,0 +1,12 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { InvalidInputError } from "../errors"
+
+export const decodeInput =
+  <S extends Schema.Top>(operation: string, schema: S) =>
+  (input: unknown) =>
+    Schema.decodeUnknownEffect(schema)(input).pipe(
+      Effect.catchTag("SchemaError", (error) =>
+        Effect.fail(InvalidInputError.make({ operation, message: error.message }))
+      )
+    )

package/src/internal/transport.ts

@@ -1,15 +1,39 @@
+import {
+  Soda3CatalogSearchResponse,
+  type Soda3ExportRequest,
+  type Soda3QueryRequest,
+  Soda3QueryResponse,
+  Soda3Row
+} from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
 import * as HttpClient from "effect/unstable/http/HttpClient"
+import type * as HttpClientError from "effect/unstable/http/HttpClientError"
 import type * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
+import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
+import { Soda3ClientConfig, type Soda3ClientConfigShape } from "../client"
 import type { Soda3ClientError } from "../errors"
 import {
   isRetryableProviderError,
+  ProviderContractError,
   ProviderHttpError,
   RetryExhaustedError,
   TimeoutError
 } from "../errors"
-import { mapHttpClientError, retryableStatus } from "./http"
-import { Soda3ClientConfig, type Soda3ClientConfigShape } from "./services"
+import {
+  httpFailureInfo,
+  httpStatusText,
+  isOkStatus,
+  retryableStatus,
+  withRequestPolicy
+} from "../transport"
+
+export { retryableStatus, toWebResponse } from "../transport"
+
+const providerRows = Schema.Array(Soda3Row)
+
+const decodeUnknown = <S extends Schema.Top>(schema: S, input: unknown) =>
+  Schema.decodeUnknownEffect(schema)(input)
 
 export const addHeaders = (config: Soda3ClientConfigShape) => ({
   accept: "application/json",
@@ -18,7 +42,83 @@ export const addHeaders = (config: Soda3ClientConfigShape) => ({
     : {})
 })
 
-export { retryableStatus }
+export const toQueryBody = (request: Soda3QueryRequest) => ({
+  query: request.query,
+  page: request.page,
+  parameters: request.parameters,
+  timeout: request.timeout,
+  includeSystem: request.includeSystem,
+  includeSynthetic: request.includeSynthetic,
+  orderingSpecifier: request.orderingSpecifier
+})
+
+export const toExportBody = (request: Soda3ExportRequest) => ({
+  query: request.query,
+  parameters: request.parameters,
+  timeout: request.timeout,
+  serializationOptions: request.serializationOptions
+})
+
+export const rangeHeader = (request: Soda3ExportRequest) =>
+  request.range === undefined
+    ? undefined
+    : `bytes=${request.range.start}-${request.range.end ?? ""}`
+
+export const mapHttpClientError = (operation: string, error: HttpClientError.HttpClientError) => {
+  const info = httpFailureInfo(error)
+
+  return ProviderHttpError.make({
+    operation,
+    status: info.status,
+    statusText: info.statusText,
+    retryable: info.status === 0 || retryableStatus(info.status)
+  })
+}
+
+export const decodeJson = (operation: string, response: HttpClientResponse.HttpClientResponse) =>
+  response.json.pipe(
+    Effect.catchTag("HttpClientError", () =>
+      Effect.fail(
+        ProviderContractError.make({
+          operation,
+          message: "Provider response was not valid JSON"
+        })
+      )
+    )
+  )
+
+export const decodeQueryResponse = (input: unknown) =>
+  decodeUnknown(Soda3QueryResponse, input).pipe(
+    Effect.catchTag("SchemaError", (error) =>
+      decodeUnknown(providerRows, input).pipe(
+        Effect.map((rows) =>
+          Soda3QueryResponse.make({
+            rows
+          })
+        ),
+        Effect.catchTag("SchemaError", () =>
+          Effect.fail(
+            ProviderContractError.make({
+              operation: "query",
+              message: error.message
+            })
+          )
+        )
+      )
+    )
+  )
+
+export const decodeCatalogSearchResponse = (input: unknown) =>
+  decodeUnknown(Soda3CatalogSearchResponse, input).pipe(
+    Effect.catchTag("SchemaError", (error) =>
+      Effect.fail(
+        ProviderContractError.make({
+          operation: "catalog",
+          message: error.message
+        })
+      )
+    )
+  )
 
 export const requestWithPolicy = <A>(
   operation: string,
@@ -26,55 +126,29 @@ export const requestWithPolicy = <A>(
 ) =>
   Effect.gen(function* () {
     const config = yield* Soda3ClientConfig
-    const retryTimes = config.retryTimes ?? 0
-    const retried =
-      retryTimes > 0
-        ? effect.pipe(
-            Effect.retry({
-              times: retryTimes,
-              while: isRetryableProviderError
-            }),
-            Effect.catchIf(isRetryableProviderError, (error) =>
-              Effect.fail(
-                RetryExhaustedError.make({
-                  operation,
-                  attempts: retryTimes + 1,
-                  message: `Retry attempts exhausted after provider status ${error.status}`
-                })
-              )
-            )
-          )
-        : effect
-
-    const timeoutMs = config.timeoutMs
-    const timed =
-      timeoutMs === undefined
-        ? retried
-        : retried.pipe(
-            Effect.timeoutOrElse({
-              duration: timeoutMs,
-              orElse: () =>
-                Effect.fail(
-                  TimeoutError.make({
-                    operation,
-                    timeoutMs
-                  })
-                )
-            })
-          )
-
-    return yield* timed
+    return yield* withRequestPolicy(config, effect, {
+      isRetryable: isRetryableProviderError,
+      onExhausted: (attempts, error) =>
+        RetryExhaustedError.make({
+          operation,
+          attempts,
+          message: `Retry attempts exhausted after provider status ${error.status}`
+        }),
+      onTimeout: (timeoutMs) =>
+        TimeoutError.make({
+          operation,
+          timeoutMs
+        })
+    })
   })
 
-const httpStatusText = (status: number) => `HTTP ${status}`
-
 export const executeRequest = (operation: string, request: HttpClientRequest.HttpClientRequest) =>
   Effect.gen(function* () {
     const response = yield* HttpClient.execute(request).pipe(
       Effect.mapError((error) => mapHttpClientError(operation, error))
     )
 
-    if (response.status < 200 || response.status >= 300) {
+    if (!isOkStatus(response.status)) {
       return yield* ProviderHttpError.make({
         operation,
         status: response.status,

package/src/metadata.ts

@@ -0,0 +1,37 @@
+import {
+  type Soda3CatalogResultFragment,
+  Soda3DatasetMetadataRequest,
+  type Soda3DatasetMetadataRequestInput
+} from "@nyc-transit-kit/contracts/soda3"
+import * as Effect from "effect/Effect"
+import { catalogSearch } from "./catalog"
+import { ProviderContractError } from "./errors"
+import { decodeInput } from "./internal/decode"
+
+export type { Soda3DatasetMetadataRequestInput }
+
+const decodeDatasetMetadataRequest = decodeInput("metadata", Soda3DatasetMetadataRequest)
+
+const matchingResult = (results: ReadonlyArray<Soda3CatalogResultFragment>, datasetId: string) =>
+  results.find((result) => String(result.resource.id) === datasetId)
+
+export const fetchDatasetMetadata = Effect.fn("Soda3.fetchDatasetMetadata")(function* (
+  input: Soda3DatasetMetadataRequestInput
+) {
+  const request = yield* decodeDatasetMetadataRequest(input)
+  const response = yield* catalogSearch({
+    domain: request.domain,
+    ids: [request.datasetId],
+    limit: 1
+  })
+  const result = matchingResult(response.results, String(request.datasetId))
+
+  if (result === undefined) {
+    return yield* ProviderContractError.make({
+      operation: "metadata",
+      message: `Discovery metadata response did not include dataset ${request.datasetId}`
+    })
+  }
+
+  return result
+})

package/src/query.ts

@@ -1,15 +1,62 @@
-import type { Soda3QueryRequestInput } from "@nyc-transit-kit/contracts/soda3"
+import {
+  PositiveInteger,
+  type Soda3QueryRequestInput,
+  type Soda3Row
+} from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
+import { Soda3ClientConfig } from "./client"
 import { buildQueryUrl, decodeSoda3QueryRequest } from "./endpoints"
-import { InvalidInputError } from "./errors"
-import { toQueryBody } from "./internal/bodies"
-import { decodeJson, decodeQueryResponse } from "./internal/responses"
-import { Soda3ClientConfig } from "./internal/services"
-import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transport"
+import { InvalidInputError, ProviderContractError } from "./errors"
+import { decodeInput } from "./internal/decode"
+import {
+  addHeaders,
+  decodeJson,
+  decodeQueryResponse,
+  executeRequest,
+  requestWithPolicy,
+  toQueryBody
+} from "./internal/transport"
 
 export type { Soda3QueryRequestInput }
 
+export type Soda3QueryAllRowsInput = Omit<Soda3QueryRequestInput, "page"> & {
+  readonly pageSize?: number
+  readonly maxRows?: number
+}
+
+export type Soda3CountRowsInput = {
+  readonly domain: string
+  readonly datasetId: string
+  readonly where?: string
+  readonly parameters?: Readonly<Record<string, unknown>>
+}
+
+const decodeHelperPositiveInteger = (name: string, value: number) =>
+  decodeInput(
+    "query",
+    PositiveInteger
+  )(value).pipe(
+    Effect.mapError((error) =>
+      InvalidInputError.make({
+        operation: "query",
+        message: `Invalid ${name}: ${error.message}`
+      })
+    )
+  )
+
+const decodeOptionalHelperPositiveInteger = (name: string, value: number | undefined) =>
+  value === undefined ? Effect.void : decodeHelperPositiveInteger(name, value)
+
+const decodeCountValue = (value: unknown) => {
+  const parsed =
+    typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN
+
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
+}
+
+const countColumnName = "count"
+
 export const queryRows = Effect.fn("Soda3.queryRows")(function* (input: Soda3QueryRequestInput) {
   const config = yield* Soda3ClientConfig
   const queryRequest = yield* decodeSoda3QueryRequest(input)
@@ -34,3 +81,58 @@ export const queryRows = Effect.fn("Soda3.queryRows")(function* (input: Soda3Que
     )
   )
 })
+
+export const queryAllRows = Effect.fn("Soda3.queryAllRows")(function* (
+  input: Soda3QueryAllRowsInput
+) {
+  const { pageSize: rawPageSize = 1000, maxRows: rawMaxRows, ...queryInput } = input
+  const pageSize = yield* decodeHelperPositiveInteger("pageSize", rawPageSize)
+  const maxRows = yield* decodeOptionalHelperPositiveInteger("maxRows", rawMaxRows)
+  const rows: Array<Soda3Row> = []
+  let pageNumber = 1
+
+  while (true) {
+    const response = yield* queryRows({
+      ...queryInput,
+      page: {
+        pageNumber,
+        pageSize
+      }
+    })
+    rows.push(...response.rows)
+
+    if (maxRows !== undefined && rows.length > maxRows) {
+      return yield* InvalidInputError.make({
+        operation: "query",
+        message: `SODA3 query exceeded maxRows=${maxRows}; narrow the query or raise maxRows`
+      })
+    }
+
+    if (response.rows.length < pageSize) {
+      return rows
+    }
+
+    pageNumber += 1
+  }
+})
+
+export const countRows = Effect.fn("Soda3.countRows")(function* (input: Soda3CountRowsInput) {
+  const query = `SELECT count(*)${input.where === undefined ? "" : ` WHERE ${input.where}`}`
+  const response = yield* queryRows({
+    domain: input.domain,
+    datasetId: input.datasetId,
+    query,
+    ...(input.parameters === undefined ? {} : { parameters: input.parameters }),
+    includeSynthetic: false
+  })
+  const count = decodeCountValue(response.rows[0]?.[countColumnName])
+
+  if (count === undefined) {
+    return yield* ProviderContractError.make({
+      operation: "query",
+      message: "SODA3 count(*) response did not include a numeric count"
+    })
+  }
+
+  return count
+})