@nyc-transit-kit/soda3 0.1.4 → 0.3.0

package/README.md

@@ -1,30 +1,47 @@
 # @nyc-transit-kit/soda3
 
-Generic Effect-native client for Socrata SODA3 query, export, and Discovery
-catalog APIs.
+Effect client for Socrata on any domain: SODA3 query and CSV export, Discovery
+catalog search, and SoQL built as `effect/sql` Statement fragments whose values
+are written as SoQL literals. It speaks SODA3 alone; the legacy row endpoints
+are not supported.
 
-Socrata-backed data access is SODA3-only. This package does not expose legacy
-Socrata endpoint helpers, fallback endpoint builders, or compatibility aliases.
-
-## Import Paths
+```sh
+bun add effect@^4.0.6 @nyc-transit-kit/soda3
+```
 
 ```ts
-import { soda3Layer } from "@nyc-transit-kit/soda3/client"
-import { queryRows } from "@nyc-transit-kit/soda3/query"
+import * as Socrata from "@nyc-transit-kit/soda3/Socrata"
+import * as Soda3Client from "@nyc-transit-kit/soda3/Soda3Client"
+import * as Soql from "@nyc-transit-kit/soda3/Soql"
 import * as Effect from "effect/Effect"
-
-const program = queryRows({
-  domain: "data.cityofnewyork.us",
-  datasetId: "ycrg-ses3",
-  query: "SELECT * LIMIT 5"
-}).pipe(Effect.provide(soda3Layer({ retryTimes: 2 })))
+import * as FetchHttpClient from "effect/http/FetchHttpClient"
+import * as Layer from "effect/Layer"
+import * as Redacted from "effect/Redacted"
+
+const rows = Effect.gen(function* () {
+  const soda = yield* Soda3Client.Soda3Client
+  return yield* soda.query({
+    domain: Socrata.SocrataDomain.make("data.ny.gov"),
+    datasetId: Socrata.SocrataDatasetId.make("f462-ka72"),
+    query: Soql.soql`SELECT dataset_title, status WHERE ${Soql.eq("agency", "New York City Transit")} LIMIT 5`
+  })
+})
+
+export const program = rows.pipe(
+  Effect.provide(
+    Soda3Client.layer({ appToken: Redacted.make("your-app-token") }).pipe(
+      Layer.provide(FetchHttpClient.layer)
+    )
+  )
+)
 ```
 
-Prefer subpaths such as `@nyc-transit-kit/soda3/query`,
-`@nyc-transit-kit/soda3/export`, `@nyc-transit-kit/soda3/catalog`, and
-`@nyc-transit-kit/soda3/client`.
-
-## Compatibility
+- SODA3 refuses query and export without an app token, so
+  `Soda3Client.layer` requires one. `Discovery.search` needs none.
+- Refusals fail with `SocrataError`, which keeps Socrata's own sentence, such
+  as "This request must be authenticated or have an application token".
+- Nothing retries or times out on its own; compose `HttpClient.retryTransient`
+  and `Effect.timeout` where you run it.
+- Modules: `/Soda3Client`, `/Dataset`, `/Discovery`, `/Soql`, `/Socrata`.
 
-Each `nyc-transit-kit` release pins one exact Effect 4 beta. Match the Effect
-version documented in `docs/getting-started.md`.
+See the [API reference](https://github.com/mannyc2/nyc-transit-kit/blob/main/docs/api-reference.md).

package/package.json

@@ -1,19 +1,16 @@
 {
   "name": "@nyc-transit-kit/soda3",
-  "version": "0.1.4",
+  "version": "0.3.0",
   "license": "MIT",
   "type": "module",
   "sideEffects": false,
   "exports": {
     ".": "./src/index.ts",
-    "./catalog": "./src/catalog.ts",
-    "./client": "./src/client.ts",
-    "./endpoints": "./src/endpoints.ts",
-    "./errors": "./src/errors.ts",
-    "./export": "./src/export.ts",
-    "./metadata": "./src/metadata.ts",
-    "./query": "./src/query.ts",
-    "./soql": "./src/soql.ts"
+    "./Dataset": "./src/Dataset.ts",
+    "./Discovery": "./src/Discovery.ts",
+    "./Socrata": "./src/Socrata.ts",
+    "./Soda3Client": "./src/Soda3Client.ts",
+    "./Soql": "./src/Soql.ts"
   },
   "types": "./src/index.ts",
   "files": [
@@ -21,17 +18,16 @@
     "README.md",
     "src"
   ],
-  "scripts": {
-    "build": "tsc -b",
-    "test": "bun test ./test"
-  },
-  "dependencies": {
-    "@nyc-transit-kit/contracts": "0.1.4",
-    "effect": "4.0.0-beta.92"
-  },
   "publishConfig": {
     "access": "public"
   },
+  "peerDependencies": {
+    "effect": "^4.0.6"
+  },
+  "devDependencies": {
+    "@nyc-transit-kit/fixtures": "0.3.0",
+    "effect": "4.0.6"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/mannyc2/nyc-transit-kit.git",

package/src/Dataset.ts

@@ -0,0 +1,72 @@
+/**
+ * Curated datasets: a Socrata dataset together with the schema of one full row,
+ * queried and decoded as typed rows.
+ */
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import * as Stream from "effect/Stream"
+import type { DatasetRef } from "./Socrata"
+import * as Soda3Client from "./Soda3Client"
+import * as Soql from "./Soql"
+
+/** A Socrata dataset and the schema of one full row. */
+export interface Dataset<
+  Row extends Schema.Codec<unknown, unknown> = Schema.Codec<unknown, unknown>
+> extends DatasetRef {
+  readonly title: string
+  readonly description: string
+  readonly row: Row
+  /** The column that orders rows in time, when there is one. `stream` pages in its order. */
+  readonly timestampColumn?: string
+}
+
+/**
+ * Typed rows. A row schema describes whole rows, so typed queries always
+ * `SELECT *`; use `Soda3Client` for projections and aggregates.
+ */
+export const query = Effect.fn("Dataset.query")(function* <
+  Row extends Schema.Codec<unknown, unknown>
+>(dataset: Dataset<Row>, clauses: Soql.Clauses = {}) {
+  const soda = yield* Soda3Client.Soda3Client
+  const rows = yield* soda.query({
+    domain: dataset.domain,
+    datasetId: dataset.datasetId,
+    query: Soql.selectAll(clauses)
+  })
+  return yield* Schema.decodeUnknownEffect(Schema.Array(dataset.row))(rows)
+})
+
+const oldestFirst = (dataset: Pick<Dataset, "timestampColumn">): Soql.OrderBy | undefined =>
+  dataset.timestampColumn === undefined ? undefined : { column: dataset.timestampColumn }
+
+/**
+ * Every matching row, one page per request, decoded as it arrives. Unless
+ * `clauses` give an `orderBy`, rows come oldest first by the dataset's
+ * `timestampColumn`, when it has one, so rows added while you page land after
+ * the pages already read. Unordered, SODA3 served Traffic Speeds newest first
+ * and page 2 repeated page 1; see `Soda3Client.streamRows`.
+ */
+export const stream = <Row extends Schema.Codec<unknown, unknown>>(
+  dataset: Dataset<Row>,
+  clauses: Omit<Soql.Clauses, "limit"> = {},
+  pageSize?: number
+) =>
+  Soda3Client.streamRows(
+    {
+      domain: dataset.domain,
+      datasetId: dataset.datasetId,
+      query: Soql.selectAll({ ...clauses, orderBy: clauses.orderBy ?? oldestFirst(dataset) })
+    },
+    pageSize
+  ).pipe(Stream.mapEffect((row) => Schema.decodeUnknownEffect(dataset.row)(row)))
+
+/** The dataset's SODA3 export as CSV bytes, narrowed by `body.query` when given. */
+export const exportBytes = (
+  dataset: DatasetRef,
+  body: Omit<Soda3Client.ExportRequest, "domain" | "datasetId"> = {}
+) =>
+  Stream.service(Soda3Client.Soda3Client).pipe(
+    Stream.flatMap((soda) =>
+      soda.export({ ...body, domain: dataset.domain, datasetId: dataset.datasetId })
+    )
+  )

package/src/Discovery.ts

@@ -0,0 +1,107 @@
+/**
+ * The Socrata Discovery API: catalog search and dataset metadata. It needs no
+ * app token.
+ */
+import * as Effect from "effect/Effect"
+import * as HttpClient from "effect/http/HttpClient"
+import * as HttpClientResponse from "effect/http/HttpClientResponse"
+import * as Schema from "effect/Schema"
+import { type DatasetRef, filterStatusOk, SocrataDatasetId, SocrataDomain } from "./Socrata"
+
+export const url = "https://api.us.socrata.com/api/catalog/v1"
+
+const Resource = Schema.Struct({
+  id: SocrataDatasetId,
+  name: Schema.String,
+  description: Schema.String,
+  type: Schema.String,
+  attribution: Schema.NullOr(Schema.String),
+  updatedAt: Schema.String,
+  createdAt: Schema.String,
+  data_updated_at: Schema.String,
+  metadata_updated_at: Schema.String,
+  columns_name: Schema.Array(Schema.String),
+  columns_field_name: Schema.Array(Schema.String),
+  columns_datatype: Schema.Array(Schema.String),
+  columns_description: Schema.Array(Schema.String)
+})
+
+export const CatalogResult = Schema.Struct({
+  resource: Resource,
+  classification: Schema.Struct({
+    categories: Schema.Array(Schema.String),
+    tags: Schema.Array(Schema.String),
+    domain_category: Schema.optionalKey(Schema.String),
+    domain_tags: Schema.Array(Schema.String)
+  }),
+  metadata: Schema.Struct({ domain: SocrataDomain }),
+  permalink: Schema.String,
+  link: Schema.String
+})
+export type CatalogResult = typeof CatalogResult.Type
+
+/** Response of `GET https://api.us.socrata.com/api/catalog/v1`. */
+export const SearchResponse = Schema.Struct({
+  results: Schema.Array(CatalogResult),
+  resultSetSize: Schema.Int
+})
+export type SearchResponse = typeof SearchResponse.Type
+
+export class DatasetNotFound extends Schema.TaggedError<DatasetNotFound>()("DatasetNotFound", {
+  domain: SocrataDomain,
+  datasetId: SocrataDatasetId
+}) {
+  override get message() {
+    return `${this.domain} has no public dataset ${this.datasetId}`
+  }
+}
+
+export interface Search {
+  readonly domain: SocrataDomain
+  readonly query?: string | undefined
+  readonly ids?: ReadonlyArray<SocrataDatasetId> | undefined
+  readonly categories?: ReadonlyArray<string> | undefined
+  readonly tags?: ReadonlyArray<string> | undefined
+  readonly order?: string | undefined
+  readonly limit?: number | undefined
+  readonly offset?: number | undefined
+}
+
+/** Searches the public datasets of one domain. */
+export const search = Effect.fn("Discovery.search")(function* (search: Search) {
+  const client = yield* HttpClient.HttpClient
+  const response = yield* client
+    .get(url, {
+      acceptJson: true,
+      urlParams: {
+        domains: search.domain,
+        search_context: search.domain,
+        only: "dataset",
+        q: search.query,
+        ids: search.ids,
+        categories: search.categories,
+        tags: search.tags,
+        order: search.order,
+        limit: search.limit,
+        offset: search.offset
+      }
+    })
+    .pipe(Effect.flatMap(filterStatusOk))
+  return yield* HttpClientResponse.schemaBodyJson(SearchResponse)(response)
+})
+
+/** The Discovery entry of one dataset. */
+export const metadata = Effect.fn("Discovery.metadata")(function* (ref: DatasetRef) {
+  const { results } = yield* search({ domain: ref.domain, ids: [ref.datasetId] })
+  const result = results.find((candidate) => candidate.resource.id === ref.datasetId)
+  return result ?? (yield* new DatasetNotFound({ domain: ref.domain, datasetId: ref.datasetId }))
+})
+
+/** Discovery lists columns as four parallel arrays; this zips them. */
+export const columns = ({ resource }: CatalogResult) =>
+  resource.columns_field_name.map((field, index) => ({
+    field,
+    name: resource.columns_name[index] ?? field,
+    datatype: resource.columns_datatype[index] ?? "",
+    description: resource.columns_description[index] ?? ""
+  }))

package/src/Socrata.ts

@@ -0,0 +1,115 @@
+/**
+ * What every Socrata API shares: domains, dataset ids, the floating timestamps
+ * in rows, and the error Socrata answers with.
+ */
+import * as Effect from "effect/Effect"
+import * as HttpClientResponse from "effect/http/HttpClientResponse"
+import * as Option from "effect/Option"
+import * as Schema from "effect/Schema"
+
+/** A Socrata domain without protocol or path, such as `data.cityofnewyork.us`. */
+export const SocrataDomain = Schema.String.pipe(
+  Schema.check(
+    Schema.isPattern(/^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/, {
+      message: "Expected a Socrata domain without a protocol or path"
+    })
+  ),
+  Schema.brand("SocrataDomain")
+)
+export type SocrataDomain = typeof SocrataDomain.Type
+
+/** The eight-character id of a dataset, such as `i4gi-tjb9`. */
+export const SocrataDatasetId = Schema.String.pipe(
+  Schema.check(
+    Schema.isPattern(/^[a-z0-9]{4}-[a-z0-9]{4}$/, {
+      message: "Expected an eight-character Socrata dataset id such as abcd-1234"
+    })
+  ),
+  Schema.brand("SocrataDatasetId")
+)
+export type SocrataDatasetId = typeof SocrataDatasetId.Type
+
+/** One dataset on one domain. */
+export interface DatasetRef {
+  readonly domain: SocrataDomain
+  readonly datasetId: SocrataDatasetId
+}
+
+/** The dataset's page, `https://{domain}/d/{id}`. */
+export const datasetUrl = (ref: DatasetRef) => `https://${ref.domain}/d/${ref.datasetId}`
+
+const isCalendarDate = (value: string) => {
+  const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value)
+  if (match === null) {
+    return false
+  }
+  const [year, month, day] = [Number(match[1]), Number(match[2]), Number(match[3])]
+  const date = new Date(Date.UTC(year, month - 1, day))
+  return (
+    date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day
+  )
+}
+
+/** A calendar date, `YYYY-MM-DD`. */
+export const IsoDate = Schema.String.pipe(
+  Schema.check(
+    Schema.makeFilter<string>(
+      (value) => isCalendarDate(value) || "Expected an ISO calendar date in YYYY-MM-DD format"
+    )
+  ),
+  Schema.brand("IsoDate")
+)
+export type IsoDate = typeof IsoDate.Type
+
+/** Floating timestamps carry no zone: `2026-10-26T23:02:06.000` is New York local time. */
+export const FloatingTimestamp = Schema.String.check(
+  Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}$/, {
+    message: "Expected a Socrata floating timestamp such as 2026-10-26T23:02:06.000"
+  })
+)
+
+export const MultiLineString = Schema.Struct({
+  type: Schema.Literal("MultiLineString"),
+  coordinates: Schema.Array(Schema.Array(Schema.Array(Schema.Finite)))
+})
+
+/** Socrata answered with a non-2xx status. `message` is Socrata's own sentence when it sent one. */
+export class SocrataError extends Schema.TaggedError<SocrataError>()("SocrataError", {
+  status: Schema.Int,
+  code: Schema.optional(Schema.String),
+  message: Schema.String
+}) {}
+
+/** Non-2xx bodies: SODA3 sends `{ code, error: true, message }`, Discovery sends `{ error }`. */
+export const SocrataErrorBody = Schema.Union([
+  Schema.Struct({ code: Schema.optionalKey(Schema.String), message: Schema.String }),
+  Schema.Struct({ error: Schema.String })
+])
+
+const decodeErrorBody = Schema.decodeUnknownOption(Schema.fromJsonString(SocrataErrorBody))
+
+const socrataError = (status: number, text: string) =>
+  Option.match(decodeErrorBody(text), {
+    onNone: () => new SocrataError({ status, message: `Socrata answered HTTP ${status}` }),
+    onSome: (body) =>
+      "message" in body
+        ? new SocrataError({ status, code: body.code, message: body.message })
+        : new SocrataError({ status, message: body.error })
+  })
+
+/**
+ * Passes 2xx responses through and fails the rest with a `SocrataError`. Unlike
+ * `HttpClient.filterStatusOk` it reads the body, which holds the only useful
+ * sentence, such as "This request must be authenticated or have an application token".
+ */
+export const filterStatusOk = (
+  response: HttpClientResponse.HttpClientResponse
+): Effect.Effect<HttpClientResponse.HttpClientResponse, SocrataError> =>
+  HttpClientResponse.matchStatus(response, {
+    "2xx": (ok) => Effect.succeed(ok),
+    orElse: (failed) =>
+      failed.text.pipe(
+        Effect.orElseSucceed(() => ""),
+        Effect.flatMap((text) => Effect.fail(socrataError(failed.status, text)))
+      )
+  })

package/src/Soda3Client.ts

@@ -0,0 +1,224 @@
+/**
+ * SODA3 `query.json` and `export.csv` on any Socrata domain.
+ */
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as HttpClient from "effect/http/HttpClient"
+import type * as HttpClientError from "effect/http/HttpClientError"
+import * as HttpClientRequest from "effect/http/HttpClientRequest"
+import * as HttpClientResponse from "effect/http/HttpClientResponse"
+import * as Layer from "effect/Layer"
+import * as Option from "effect/Option"
+import * as Redacted from "effect/Redacted"
+import * as Schema from "effect/Schema"
+import * as Stream from "effect/Stream"
+import * as Struct from "effect/Struct"
+import { type DatasetRef, filterStatusOk, type SocrataError } from "./Socrata"
+import * as Soql from "./Soql"
+
+export const OrderingSpecifier = Schema.Literals(["total", "discard"])
+export type OrderingSpecifier = typeof OrderingSpecifier.Type
+
+export const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0))
+
+export const Page = Schema.Struct({ pageNumber: PositiveInt, pageSize: PositiveInt })
+export type Page = typeof Page.Type
+
+/** Only parameterized views read `parameters`; ordinary queries carry their values in the SoQL. */
+export const Parameters = Schema.Record(
+  Schema.String,
+  Schema.Union([Schema.String, Schema.Finite, Schema.Boolean])
+)
+
+/** Body of `POST /api/v3/views/{id}/query.json`. */
+export const QueryBody = Schema.Struct({
+  query: Schema.String,
+  page: Schema.optionalKey(Page),
+  parameters: Schema.optionalKey(Parameters),
+  timeout: Schema.optionalKey(PositiveInt),
+  includeSystem: Schema.optionalKey(Schema.Boolean),
+  includeSynthetic: Schema.optionalKey(Schema.Boolean),
+  orderingSpecifier: Schema.optionalKey(OrderingSpecifier)
+})
+export type QueryBody = typeof QueryBody.Type
+
+/**
+ * Body of `POST /api/v3/views/{id}/export.csv`. SODA3's documentation shapes
+ * the file with `serializationOptions`, such as `{ separator: "\t", bom: true }`.
+ */
+export const ExportBody = Schema.Struct({
+  query: Schema.optionalKey(Schema.String),
+  parameters: Schema.optionalKey(Parameters),
+  timeout: Schema.optionalKey(PositiveInt),
+  orderingSpecifier: Schema.optionalKey(OrderingSpecifier),
+  serializationOptions: Schema.optionalKey(
+    Schema.Record(Schema.String, Schema.Union([Schema.String, Schema.Boolean]))
+  )
+})
+export type ExportBody = typeof ExportBody.Type
+
+export const Row = Schema.Record(Schema.String, Schema.Unknown)
+export type Row = typeof Row.Type
+
+/**
+ * A bare array of rows. Numbers arrive as text, null cells are left out, and
+ * unless `includeSystem` or `includeSynthetic` is false each row ends with the
+ * system columns `:id`, `:version`, `:created_at` and `:updated_at`.
+ */
+export const QueryResponse = Schema.Array(Row)
+
+/** A query: the body with SoQL as text or as a fragment. */
+export interface QueryRequest extends DatasetRef, Omit<QueryBody, "query"> {
+  readonly query: string | Soql.Fragment
+}
+
+/**
+ * A CSV export, narrowed by `query` when given. CSV is the format SODA3
+ * exported on 2026-11-01; it refused `json` and `geojson` with a 406 on every
+ * dataset tried.
+ */
+export interface ExportRequest extends DatasetRef, Omit<ExportBody, "query"> {
+  readonly query?: string | Soql.Fragment
+  /**
+   * Ask for a byte range. On 2026-11-01 SODA3 ignored it and sent the whole
+   * export with a 200; `ntk socrata range-probe` checks again.
+   */
+  readonly range?: { readonly start: number; readonly end?: number }
+}
+
+export type QueryError =
+  | HttpClientError.HttpClientError
+  | SocrataError
+  | Soql.SoqlError
+  | Schema.SchemaError
+
+export type ExportError = HttpClientError.HttpClientError | SocrataError | Soql.SoqlError
+
+export const viewUrl = (ref: DatasetRef, file: string) =>
+  `https://${ref.domain}/api/v3/views/${ref.datasetId}/${file}`
+
+/**
+ * SODA3 query, and export as CSV. Socrata refuses both without an app token,
+ * so the layer requires one. Retries and timeouts belong to the caller's
+ * HttpClient. SODA3 has no default limit: a query without `LIMIT` or `page`
+ * returns every matching row.
+ */
+export class Soda3Client extends Context.Service<
+  Soda3Client,
+  {
+    readonly query: (request: QueryRequest) => Effect.Effect<ReadonlyArray<Row>, QueryError>
+    readonly export: (request: ExportRequest) => Stream.Stream<Uint8Array, ExportError>
+  }
+>()("@nyc-transit-kit/soda3/Soda3Client") {}
+
+const soqlText = (query: string | Soql.Fragment) =>
+  typeof query === "string" ? Effect.succeed(query) : Soql.compile(query)
+
+// Only these fields reach the body, so a request built by spreading a richer
+// object, such as a curated `Dataset`, sends nothing else.
+const queryOptions = [
+  "page",
+  "parameters",
+  "timeout",
+  "includeSystem",
+  "includeSynthetic",
+  "orderingSpecifier"
+] as const
+const exportOptions = [
+  "parameters",
+  "timeout",
+  "orderingSpecifier",
+  "serializationOptions"
+] as const
+
+const exportBody = Effect.fnUntraced(function* (request: ExportRequest) {
+  const body = Struct.pick(request, exportOptions)
+  return request.query === undefined ? body : { ...body, query: yield* soqlText(request.query) }
+})
+
+export const layer = (options: { readonly appToken: Redacted.Redacted }) =>
+  Layer.effect(
+    Soda3Client,
+    Effect.gen(function* () {
+      const client = (yield* HttpClient.HttpClient).pipe(
+        HttpClient.mapRequest(
+          HttpClientRequest.setHeader("x-app-token", Redacted.value(options.appToken))
+        ),
+        HttpClient.transformResponse(Effect.flatMap(filterStatusOk))
+      )
+      const post = (url: string, body: object, headers: Readonly<Record<string, string>> = {}) =>
+        client.execute(
+          HttpClientRequest.post(url, { headers }).pipe(HttpClientRequest.bodyJsonUnsafe(body))
+        )
+
+      const query = Effect.fn("Soda3Client.query")(function* (request: QueryRequest) {
+        const response = yield* post(
+          viewUrl(request, "query.json"),
+          { ...Struct.pick(request, queryOptions), query: yield* soqlText(request.query) },
+          { accept: "application/json" }
+        )
+        return yield* HttpClientResponse.schemaBodyJson(QueryResponse)(response)
+      })
+
+      const exportBytes = (request: ExportRequest) =>
+        Stream.unwrap(
+          Effect.map(exportBody(request), (body) =>
+            HttpClientResponse.stream(
+              post(
+                viewUrl(request, "export.csv"),
+                body,
+                request.range === undefined
+                  ? {}
+                  : { range: `bytes=${request.range.start}-${request.range.end ?? ""}` }
+              )
+            )
+          )
+        ).pipe(Stream.withSpan("Soda3Client.export"))
+
+      return Soda3Client.of({ query, export: exportBytes })
+    })
+  )
+
+/**
+ * Every row of a query, one page per request. Pages end at the first short page.
+ *
+ * A page is a slice of the query's order, so order by something that puts new
+ * rows last, such as the dataset's timestamp column, when the dataset changes
+ * while you page. Without an `ORDER BY`, SODA3 served Traffic Speeds newest
+ * first, requests seconds apart disagreed on the newest rows, and page 2
+ * repeated page 1. SODA3 refuses paging with `orderingSpecifier: "discard"`,
+ * so this takes none.
+ */
+export const streamRows = (
+  request: Omit<QueryRequest, "page" | "orderingSpecifier">,
+  pageSize = 1000
+) =>
+  Stream.paginate(1, (pageNumber: number) =>
+    Effect.gen(function* () {
+      const soda = yield* Soda3Client
+      const rows = yield* soda.query({ ...request, page: { pageNumber, pageSize } })
+      const next = rows.length < pageSize ? Option.none<number>() : Option.some(pageNumber + 1)
+      return [rows, next] as const
+    })
+  )
+
+// SODA3 sends `count(*)` as text, like every number in a row.
+const CountRows = Schema.Tuple([Schema.Struct({ count: Schema.FiniteFromString })])
+
+/** `count(*)` of the rows that match `where`, or of all rows. */
+export const countRows = Effect.fn("Soda3Client.countRows")(function* (
+  ref: DatasetRef,
+  where?: Soql.Fragment
+) {
+  const soda = yield* Soda3Client
+  const rows = yield* soda.query({
+    ...ref,
+    query:
+      where === undefined
+        ? Soql.soql`SELECT count(*) AS count`
+        : Soql.soql`SELECT count(*) AS count WHERE ${where}`,
+    includeSynthetic: false
+  })
+  const [row] = yield* Schema.decodeUnknownEffect(CountRows)(rows)
+  return row.count
+})