@nyc-transit-kit/soda3 0.1.3 → 0.1.4

package/README.md

@@ -1,5 +1,30 @@
 # @nyc-transit-kit/soda3
 
-Part of `nyc-transit-kit`, an Effect-native Bun toolkit for official NYC and MTA transit data APIs.
+Generic Effect-native client for Socrata SODA3 query, export, and Discovery
+catalog APIs.
 
-See the repository README and docs/api-reference.md for public import paths, CLI commands, and release notes.
+Socrata-backed data access is SODA3-only. This package does not expose legacy
+Socrata endpoint helpers, fallback endpoint builders, or compatibility aliases.
+
+## Import Paths
+
+```ts
+import { soda3Layer } from "@nyc-transit-kit/soda3/client"
+import { queryRows } from "@nyc-transit-kit/soda3/query"
+import * as Effect from "effect/Effect"
+
+const program = queryRows({
+  domain: "data.cityofnewyork.us",
+  datasetId: "ycrg-ses3",
+  query: "SELECT * LIMIT 5"
+}).pipe(Effect.provide(soda3Layer({ retryTimes: 2 })))
+```
+
+Prefer subpaths such as `@nyc-transit-kit/soda3/query`,
+`@nyc-transit-kit/soda3/export`, `@nyc-transit-kit/soda3/catalog`, and
+`@nyc-transit-kit/soda3/client`.
+
+## Compatibility
+
+Each `nyc-transit-kit` release pins one exact Effect 4 beta. Match the Effect
+version documented in `docs/getting-started.md`.

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@nyc-transit-kit/soda3",
-  "version": "0.1.3",
+  "version": "0.1.4",
+  "license": "MIT",
   "type": "module",
   "sideEffects": false,
   "exports": {
@@ -10,6 +11,7 @@
     "./endpoints": "./src/endpoints.ts",
     "./errors": "./src/errors.ts",
     "./export": "./src/export.ts",
+    "./metadata": "./src/metadata.ts",
     "./query": "./src/query.ts",
     "./soql": "./src/soql.ts"
   },
@@ -24,13 +26,12 @@
     "test": "bun test ./test"
   },
   "dependencies": {
-    "@nyc-transit-kit/contracts": "0.1.3",
+    "@nyc-transit-kit/contracts": "0.1.4",
     "effect": "4.0.0-beta.92"
   },
   "publishConfig": {
     "access": "public"
   },
-  "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mannyc2/nyc-transit-kit.git",

package/src/client.ts

@@ -1,7 +1,11 @@
 export { catalogSearch } from "./catalog"
 export { exportResponse } from "./export"
 export { Soda3HttpLive } from "./internal/http"
-export type { Soda3ClientConfigShape, Soda3LayerOptions } from "./internal/services"
+export type {
+  FetchImplementation,
+  Soda3ClientConfigShape,
+  Soda3LayerOptions
+} from "./internal/services"
 export {
   Soda3ClientConfig,
   Soda3Live,
@@ -9,4 +13,5 @@ export {
   soda3FetchLayer,
   soda3Layer
 } from "./internal/services"
-export { queryRows } from "./query"
+export { fetchDatasetMetadata } from "./metadata"
+export { countRows, queryAllRows, queryRows } from "./query"

package/src/endpoints.ts

@@ -83,6 +83,18 @@ export const buildCatalogSearchUrl = Effect.fn("Soda3.buildCatalogSearchUrl")(fu
   if (request.query !== undefined) {
     url.searchParams.set("search", request.query)
   }
+  for (const id of request.ids ?? []) {
+    url.searchParams.append("ids", id)
+  }
+  for (const category of request.categories ?? []) {
+    url.searchParams.append("categories", category)
+  }
+  for (const tag of request.tags ?? []) {
+    url.searchParams.append("tags", tag)
+  }
+  if (request.order !== undefined) {
+    url.searchParams.set("order", request.order)
+  }
   if (request.limit !== undefined) {
     url.searchParams.set("limit", request.limit.toString())
   }

package/src/index.ts

@@ -1,6 +1,6 @@
 export const packageName = "@nyc-transit-kit/soda3"
 export { buildCatalogSearchUrl, catalogSearch } from "./catalog"
-export type { Soda3ClientConfigShape, Soda3LayerOptions } from "./client"
+export type { FetchImplementation, Soda3ClientConfigShape, Soda3LayerOptions } from "./client"
 export {
   Soda3ClientConfig,
   Soda3HttpLive,
@@ -30,7 +30,10 @@ export {
   TimeoutError
 } from "./errors"
 export { exportResponse } from "./export"
-export { queryRows } from "./query"
+export type { Soda3DatasetMetadataRequestInput } from "./metadata"
+export { fetchDatasetMetadata } from "./metadata"
+export type { Soda3CountRowsInput, Soda3QueryAllRowsInput } from "./query"
+export { countRows, queryAllRows, queryRows } from "./query"
 export type {
   SocrataTimestampWindow,
   SocrataYearMonth,

package/src/internal/bodies.ts

@@ -18,4 +18,6 @@ export const toExportBody = (request: Soda3ExportRequest) => ({
 })
 
 export const rangeHeader = (request: Soda3ExportRequest) =>
-  request.range === undefined ? undefined : `bytes=${request.range.start}-${request.range.end}`
+  request.range === undefined
+    ? undefined
+    : `bytes=${request.range.start}-${request.range.end ?? ""}`

package/src/internal/services.ts

@@ -6,11 +6,17 @@ import { Soda3HttpLive } from "./http"
 export interface Soda3ClientConfigShape {
   readonly appToken?: string
   readonly retryTimes?: number
+  readonly retryDelayMs?: number
   readonly timeoutMs?: number
 }
 
+export type FetchImplementation = (
+  input: string | URL | globalThis.Request,
+  init?: globalThis.RequestInit
+) => Promise<globalThis.Response>
+
 export interface Soda3LayerOptions extends Soda3ClientConfigShape {
-  readonly fetch?: typeof fetch
+  readonly fetch?: FetchImplementation
 }
 
 export class Soda3ClientConfig extends Context.Service<Soda3ClientConfig, Soda3ClientConfigShape>()(
@@ -25,14 +31,15 @@ export const soda3ConfigLayer = (options: Soda3ClientConfigShape = {}) =>
   Layer.succeed(Soda3ClientConfig)({
     retryTimes: options.retryTimes ?? 0,
     ...(options.appToken === undefined ? {} : { appToken: options.appToken }),
+    ...(options.retryDelayMs === undefined ? {} : { retryDelayMs: options.retryDelayMs }),
     ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs })
   })
 
-export const soda3FetchLayer = (fetchImplementation?: typeof fetch) =>
+export const soda3FetchLayer = (fetchImplementation?: FetchImplementation) =>
   fetchImplementation === undefined
     ? Soda3HttpLive
     : FetchHttpClient.layer.pipe(
-        Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fetchImplementation))
+        Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fetchImplementation as typeof fetch))
       )
 
 export const soda3Layer = (options: Soda3LayerOptions = {}) =>

package/src/internal/transport.ts

@@ -1,4 +1,6 @@
+import * as Duration from "effect/Duration"
 import * as Effect from "effect/Effect"
+import * as Schedule from "effect/Schedule"
 import * as HttpClient from "effect/unstable/http/HttpClient"
 import type * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
 import type { Soda3ClientError } from "../errors"
@@ -20,6 +22,19 @@ export const addHeaders = (config: Soda3ClientConfigShape) => ({
 
 export { retryableStatus }
 
+const retryPolicy = (retryTimes: number, retryDelayMs: number | undefined) =>
+  retryDelayMs === undefined || retryDelayMs <= 0
+    ? {
+        times: retryTimes,
+        while: isRetryableProviderError
+      }
+    : {
+        schedule: Schedule.exponential(Duration.millis(retryDelayMs)).pipe(
+          Schedule.both(Schedule.recurs(retryTimes))
+        ),
+        while: isRetryableProviderError
+      }
+
 export const requestWithPolicy = <A>(
   operation: string,
   effect: Effect.Effect<A, Soda3ClientError, HttpClient.HttpClient | Soda3ClientConfig>
@@ -30,10 +45,7 @@ export const requestWithPolicy = <A>(
     const retried =
       retryTimes > 0
         ? effect.pipe(
-            Effect.retry({
-              times: retryTimes,
-              while: isRetryableProviderError
-            }),
+            Effect.retry(retryPolicy(retryTimes, config.retryDelayMs)),
             Effect.catchIf(isRetryableProviderError, (error) =>
               Effect.fail(
                 RetryExhaustedError.make({

package/src/metadata.ts

@@ -0,0 +1,47 @@
+import {
+  type Soda3CatalogResultFragment,
+  Soda3DatasetMetadataRequest,
+  type Soda3DatasetMetadataRequestInput
+} from "@nyc-transit-kit/contracts/soda3"
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { catalogSearch } from "./catalog"
+import { InvalidInputError, ProviderContractError } from "./errors"
+
+export type { Soda3DatasetMetadataRequestInput }
+
+const decodeDatasetMetadataRequest = (input: unknown) =>
+  Schema.decodeUnknownEffect(Soda3DatasetMetadataRequest)(input).pipe(
+    Effect.catchTag("SchemaError", (error) =>
+      Effect.fail(
+        InvalidInputError.make({
+          operation: "metadata",
+          message: error.message
+        })
+      )
+    )
+  )
+
+const matchingResult = (results: ReadonlyArray<Soda3CatalogResultFragment>, datasetId: string) =>
+  results.find((result) => String(result.resource.id) === datasetId)
+
+export const fetchDatasetMetadata = Effect.fn("Soda3.fetchDatasetMetadata")(function* (
+  input: Soda3DatasetMetadataRequestInput
+) {
+  const request = yield* decodeDatasetMetadataRequest(input)
+  const response = yield* catalogSearch({
+    domain: request.domain,
+    ids: [request.datasetId],
+    limit: 1
+  })
+  const result = matchingResult(response.results, String(request.datasetId))
+
+  if (result === undefined) {
+    return yield* ProviderContractError.make({
+      operation: "metadata",
+      message: `Discovery metadata response did not include dataset ${request.datasetId}`
+    })
+  }
+
+  return result
+})

package/src/query.ts

@@ -1,8 +1,13 @@
-import type { Soda3QueryRequestInput } from "@nyc-transit-kit/contracts/soda3"
+import {
+  PositiveInteger,
+  type Soda3QueryRequestInput,
+  type Soda3Row
+} from "@nyc-transit-kit/contracts/soda3"
 import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
 import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
 import { buildQueryUrl, decodeSoda3QueryRequest } from "./endpoints"
-import { InvalidInputError } from "./errors"
+import { InvalidInputError, ProviderContractError } from "./errors"
 import { toQueryBody } from "./internal/bodies"
 import { decodeJson, decodeQueryResponse } from "./internal/responses"
 import { Soda3ClientConfig } from "./internal/services"
@@ -10,6 +15,42 @@ import { addHeaders, executeRequest, requestWithPolicy } from "./internal/transp
 
 export type { Soda3QueryRequestInput }
 
+export type Soda3QueryAllRowsInput = Omit<Soda3QueryRequestInput, "page"> & {
+  readonly pageSize?: number
+  readonly maxRows?: number
+}
+
+export type Soda3CountRowsInput = {
+  readonly domain: string
+  readonly datasetId: string
+  readonly where?: string
+  readonly parameters?: Readonly<Record<string, unknown>>
+}
+
+const decodeHelperPositiveInteger = (name: string, value: number) =>
+  Schema.decodeUnknownEffect(PositiveInteger)(value).pipe(
+    Effect.catchTag("SchemaError", (error) =>
+      Effect.fail(
+        InvalidInputError.make({
+          operation: "query",
+          message: `Invalid ${name}: ${error.message}`
+        })
+      )
+    )
+  )
+
+const decodeOptionalHelperPositiveInteger = (name: string, value: number | undefined) =>
+  value === undefined ? Effect.void : decodeHelperPositiveInteger(name, value)
+
+const decodeCountValue = (value: unknown) => {
+  const parsed =
+    typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN
+
+  return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
+}
+
+const countColumnName = "count"
+
 export const queryRows = Effect.fn("Soda3.queryRows")(function* (input: Soda3QueryRequestInput) {
   const config = yield* Soda3ClientConfig
   const queryRequest = yield* decodeSoda3QueryRequest(input)
@@ -34,3 +75,58 @@ export const queryRows = Effect.fn("Soda3.queryRows")(function* (input: Soda3Que
     )
   )
 })
+
+export const queryAllRows = Effect.fn("Soda3.queryAllRows")(function* (
+  input: Soda3QueryAllRowsInput
+) {
+  const { pageSize: rawPageSize = 1000, maxRows: rawMaxRows, ...queryInput } = input
+  const pageSize = yield* decodeHelperPositiveInteger("pageSize", rawPageSize)
+  const maxRows = yield* decodeOptionalHelperPositiveInteger("maxRows", rawMaxRows)
+  const rows: Array<Soda3Row> = []
+  let pageNumber = 1
+
+  while (true) {
+    const response = yield* queryRows({
+      ...queryInput,
+      page: {
+        pageNumber,
+        pageSize
+      }
+    })
+    rows.push(...response.rows)
+
+    if (maxRows !== undefined && rows.length > maxRows) {
+      return yield* InvalidInputError.make({
+        operation: "query",
+        message: `SODA3 query exceeded maxRows=${maxRows}; narrow the query or raise maxRows`
+      })
+    }
+
+    if (response.rows.length < pageSize) {
+      return rows
+    }
+
+    pageNumber += 1
+  }
+})
+
+export const countRows = Effect.fn("Soda3.countRows")(function* (input: Soda3CountRowsInput) {
+  const query = `SELECT count(*)${input.where === undefined ? "" : ` WHERE ${input.where}`}`
+  const response = yield* queryRows({
+    domain: input.domain,
+    datasetId: input.datasetId,
+    query,
+    ...(input.parameters === undefined ? {} : { parameters: input.parameters }),
+    includeSynthetic: false
+  })
+  const count = decodeCountValue(response.rows[0]?.[countColumnName])
+
+  if (count === undefined) {
+    return yield* ProviderContractError.make({
+      operation: "query",
+      message: "SODA3 count(*) response did not include a numeric count"
+    })
+  }
+
+  return count
+})