@nyc-transit-kit/soda3 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/catalog.ts +8 -3
- package/src/client.ts +40 -13
- package/src/endpoints.ts +4 -37
- package/src/export.ts +9 -4
- package/src/index.ts +12 -0
- package/src/internal/decode.ts +12 -0
- package/src/internal/transport.ts +117 -55
- package/src/metadata.ts +3 -13
- package/src/query.ts +19 -13
- package/src/soql.ts +124 -263
- package/src/transport.ts +145 -0
- package/src/internal/bodies.ts +0 -23
- package/src/internal/http.ts +0 -42
- package/src/internal/responses.ts +0 -56
- package/src/internal/services.ts +0 -48
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nyc-transit-kit/soda3",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
"./export": "./src/export.ts",
|
|
14
14
|
"./metadata": "./src/metadata.ts",
|
|
15
15
|
"./query": "./src/query.ts",
|
|
16
|
-
"./soql": "./src/soql.ts"
|
|
16
|
+
"./soql": "./src/soql.ts",
|
|
17
|
+
"./transport": "./src/transport.ts"
|
|
17
18
|
},
|
|
18
19
|
"types": "./src/index.ts",
|
|
19
20
|
"files": [
|
|
@@ -26,7 +27,7 @@
|
|
|
26
27
|
"test": "bun test ./test"
|
|
27
28
|
},
|
|
28
29
|
"dependencies": {
|
|
29
|
-
"@nyc-transit-kit/contracts": "0.
|
|
30
|
+
"@nyc-transit-kit/contracts": "0.2.0",
|
|
30
31
|
"effect": "4.0.0-beta.92"
|
|
31
32
|
},
|
|
32
33
|
"publishConfig": {
|
package/src/catalog.ts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import type { Soda3CatalogSearchRequestInput } from "@nyc-transit-kit/contracts/soda3"
|
|
2
2
|
import * as Effect from "effect/Effect"
|
|
3
3
|
import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
4
|
+
import { Soda3ClientConfig } from "./client"
|
|
4
5
|
import { buildCatalogSearchUrl } from "./endpoints"
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
import {
|
|
7
|
+
addHeaders,
|
|
8
|
+
decodeCatalogSearchResponse,
|
|
9
|
+
decodeJson,
|
|
10
|
+
executeRequest,
|
|
11
|
+
requestWithPolicy
|
|
12
|
+
} from "./internal/transport"
|
|
8
13
|
|
|
9
14
|
export { buildCatalogSearchUrl } from "./endpoints"
|
|
10
15
|
export type { Soda3CatalogSearchRequestInput }
|
package/src/client.ts
CHANGED
|
@@ -1,17 +1,44 @@
|
|
|
1
|
+
import * as Context from "effect/Context"
|
|
2
|
+
import * as Layer from "effect/Layer"
|
|
3
|
+
import { type FetchImplementation, fetchHttpClientLayer } from "./transport"
|
|
4
|
+
|
|
1
5
|
export { catalogSearch } from "./catalog"
|
|
2
6
|
export { exportResponse } from "./export"
|
|
3
|
-
export { Soda3HttpLive } from "./internal/http"
|
|
4
|
-
export type {
|
|
5
|
-
FetchImplementation,
|
|
6
|
-
Soda3ClientConfigShape,
|
|
7
|
-
Soda3LayerOptions
|
|
8
|
-
} from "./internal/services"
|
|
9
|
-
export {
|
|
10
|
-
Soda3ClientConfig,
|
|
11
|
-
Soda3Live,
|
|
12
|
-
soda3ConfigLayer,
|
|
13
|
-
soda3FetchLayer,
|
|
14
|
-
soda3Layer
|
|
15
|
-
} from "./internal/services"
|
|
16
7
|
export { fetchDatasetMetadata } from "./metadata"
|
|
17
8
|
export { countRows, queryAllRows, queryRows } from "./query"
|
|
9
|
+
export type { FetchImplementation } from "./transport"
|
|
10
|
+
|
|
11
|
+
export interface Soda3ClientConfigShape {
|
|
12
|
+
readonly appToken?: string
|
|
13
|
+
readonly retryTimes?: number
|
|
14
|
+
readonly retryDelayMs?: number
|
|
15
|
+
readonly timeoutMs?: number
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface Soda3LayerOptions extends Soda3ClientConfigShape {
|
|
19
|
+
readonly fetch?: FetchImplementation
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export class Soda3ClientConfig extends Context.Service<Soda3ClientConfig, Soda3ClientConfigShape>()(
|
|
23
|
+
"@nyc-transit-kit/soda3/Soda3ClientConfig"
|
|
24
|
+
) {
|
|
25
|
+
static readonly Default = Layer.succeed(Soda3ClientConfig)({
|
|
26
|
+
retryTimes: 0
|
|
27
|
+
})
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const soda3ConfigLayer = (options: Soda3ClientConfigShape = {}) =>
|
|
31
|
+
Layer.succeed(Soda3ClientConfig)({
|
|
32
|
+
retryTimes: options.retryTimes ?? 0,
|
|
33
|
+
...(options.appToken === undefined ? {} : { appToken: options.appToken }),
|
|
34
|
+
...(options.retryDelayMs === undefined ? {} : { retryDelayMs: options.retryDelayMs }),
|
|
35
|
+
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs })
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
export const Soda3HttpLive = fetchHttpClientLayer()
|
|
39
|
+
export const soda3FetchLayer = fetchHttpClientLayer
|
|
40
|
+
|
|
41
|
+
export const soda3Layer = (options: Soda3LayerOptions = {}) =>
|
|
42
|
+
Layer.mergeAll(soda3ConfigLayer(options), soda3FetchLayer(options.fetch))
|
|
43
|
+
|
|
44
|
+
export const Soda3Live = soda3Layer()
|
package/src/endpoints.ts
CHANGED
|
@@ -7,48 +7,15 @@ import {
|
|
|
7
7
|
type Soda3QueryRequestInput
|
|
8
8
|
} from "@nyc-transit-kit/contracts/soda3"
|
|
9
9
|
import * as Effect from "effect/Effect"
|
|
10
|
-
import
|
|
11
|
-
import { InvalidInputError } from "./errors"
|
|
10
|
+
import { decodeInput } from "./internal/decode"
|
|
12
11
|
|
|
13
12
|
export const socrataApiVersion = "v3"
|
|
14
13
|
export const defaultSocrataProtocol = "https"
|
|
15
14
|
export const discoveryApiHost = "api.us.socrata.com"
|
|
16
15
|
|
|
17
|
-
const decodeQueryRequest = (
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
Effect.fail(
|
|
21
|
-
InvalidInputError.make({
|
|
22
|
-
operation: "query",
|
|
23
|
-
message: error.message
|
|
24
|
-
})
|
|
25
|
-
)
|
|
26
|
-
)
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
const decodeExportRequest = (input: unknown) =>
|
|
30
|
-
Schema.decodeUnknownEffect(Soda3ExportRequest)(input).pipe(
|
|
31
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
32
|
-
Effect.fail(
|
|
33
|
-
InvalidInputError.make({
|
|
34
|
-
operation: "export",
|
|
35
|
-
message: error.message
|
|
36
|
-
})
|
|
37
|
-
)
|
|
38
|
-
)
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
const decodeCatalogSearchRequest = (input: unknown) =>
|
|
42
|
-
Schema.decodeUnknownEffect(Soda3CatalogSearchRequest)(input).pipe(
|
|
43
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
44
|
-
Effect.fail(
|
|
45
|
-
InvalidInputError.make({
|
|
46
|
-
operation: "catalog",
|
|
47
|
-
message: error.message
|
|
48
|
-
})
|
|
49
|
-
)
|
|
50
|
-
)
|
|
51
|
-
)
|
|
16
|
+
const decodeQueryRequest = decodeInput("query", Soda3QueryRequest)
|
|
17
|
+
const decodeExportRequest = decodeInput("export", Soda3ExportRequest)
|
|
18
|
+
const decodeCatalogSearchRequest = decodeInput("catalog", Soda3CatalogSearchRequest)
|
|
52
19
|
|
|
53
20
|
const viewUrl = (domain: string, datasetId: string, suffix: string) => {
|
|
54
21
|
const url = new URL(`${defaultSocrataProtocol}://${domain}`)
|
package/src/export.ts
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import type { Soda3ExportRequestInput } from "@nyc-transit-kit/contracts/soda3"
|
|
2
2
|
import * as Effect from "effect/Effect"
|
|
3
3
|
import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
4
|
+
import { Soda3ClientConfig } from "./client"
|
|
4
5
|
import { buildExportUrl, decodeSoda3ExportRequest } from "./endpoints"
|
|
5
6
|
import { InvalidInputError } from "./errors"
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
import {
|
|
8
|
+
addHeaders,
|
|
9
|
+
executeRequest,
|
|
10
|
+
rangeHeader,
|
|
11
|
+
requestWithPolicy,
|
|
12
|
+
toExportBody,
|
|
13
|
+
toWebResponse
|
|
14
|
+
} from "./internal/transport"
|
|
10
15
|
|
|
11
16
|
export type { Soda3ExportRequestInput }
|
|
12
17
|
|
package/src/index.ts
CHANGED
|
@@ -59,3 +59,15 @@ export {
|
|
|
59
59
|
soqlTimestampRange,
|
|
60
60
|
soqlYearMonthRange
|
|
61
61
|
} from "./soql"
|
|
62
|
+
export type { RequestPolicyConfig } from "./transport"
|
|
63
|
+
export {
|
|
64
|
+
contentLengthOf,
|
|
65
|
+
fetchHttpClientLayer,
|
|
66
|
+
httpFailureInfo,
|
|
67
|
+
httpStatusText,
|
|
68
|
+
isOkStatus,
|
|
69
|
+
redactedUrl,
|
|
70
|
+
retryableStatus,
|
|
71
|
+
toWebResponse,
|
|
72
|
+
withRequestPolicy
|
|
73
|
+
} from "./transport"
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import * as Effect from "effect/Effect"
|
|
2
|
+
import * as Schema from "effect/Schema"
|
|
3
|
+
import { InvalidInputError } from "../errors"
|
|
4
|
+
|
|
5
|
+
export const decodeInput =
|
|
6
|
+
<S extends Schema.Top>(operation: string, schema: S) =>
|
|
7
|
+
(input: unknown) =>
|
|
8
|
+
Schema.decodeUnknownEffect(schema)(input).pipe(
|
|
9
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
10
|
+
Effect.fail(InvalidInputError.make({ operation, message: error.message }))
|
|
11
|
+
)
|
|
12
|
+
)
|
|
@@ -1,17 +1,39 @@
|
|
|
1
|
-
import
|
|
1
|
+
import {
|
|
2
|
+
Soda3CatalogSearchResponse,
|
|
3
|
+
type Soda3ExportRequest,
|
|
4
|
+
type Soda3QueryRequest,
|
|
5
|
+
Soda3QueryResponse,
|
|
6
|
+
Soda3Row
|
|
7
|
+
} from "@nyc-transit-kit/contracts/soda3"
|
|
2
8
|
import * as Effect from "effect/Effect"
|
|
3
|
-
import * as
|
|
9
|
+
import * as Schema from "effect/Schema"
|
|
4
10
|
import * as HttpClient from "effect/unstable/http/HttpClient"
|
|
11
|
+
import type * as HttpClientError from "effect/unstable/http/HttpClientError"
|
|
5
12
|
import type * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
13
|
+
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
14
|
+
import { Soda3ClientConfig, type Soda3ClientConfigShape } from "../client"
|
|
6
15
|
import type { Soda3ClientError } from "../errors"
|
|
7
16
|
import {
|
|
8
17
|
isRetryableProviderError,
|
|
18
|
+
ProviderContractError,
|
|
9
19
|
ProviderHttpError,
|
|
10
20
|
RetryExhaustedError,
|
|
11
21
|
TimeoutError
|
|
12
22
|
} from "../errors"
|
|
13
|
-
import {
|
|
14
|
-
|
|
23
|
+
import {
|
|
24
|
+
httpFailureInfo,
|
|
25
|
+
httpStatusText,
|
|
26
|
+
isOkStatus,
|
|
27
|
+
retryableStatus,
|
|
28
|
+
withRequestPolicy
|
|
29
|
+
} from "../transport"
|
|
30
|
+
|
|
31
|
+
export { retryableStatus, toWebResponse } from "../transport"
|
|
32
|
+
|
|
33
|
+
const providerRows = Schema.Array(Soda3Row)
|
|
34
|
+
|
|
35
|
+
const decodeUnknown = <S extends Schema.Top>(schema: S, input: unknown) =>
|
|
36
|
+
Schema.decodeUnknownEffect(schema)(input)
|
|
15
37
|
|
|
16
38
|
export const addHeaders = (config: Soda3ClientConfigShape) => ({
|
|
17
39
|
accept: "application/json",
|
|
@@ -20,20 +42,83 @@ export const addHeaders = (config: Soda3ClientConfigShape) => ({
|
|
|
20
42
|
: {})
|
|
21
43
|
})
|
|
22
44
|
|
|
23
|
-
export
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
45
|
+
export const toQueryBody = (request: Soda3QueryRequest) => ({
|
|
46
|
+
query: request.query,
|
|
47
|
+
page: request.page,
|
|
48
|
+
parameters: request.parameters,
|
|
49
|
+
timeout: request.timeout,
|
|
50
|
+
includeSystem: request.includeSystem,
|
|
51
|
+
includeSynthetic: request.includeSynthetic,
|
|
52
|
+
orderingSpecifier: request.orderingSpecifier
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
export const toExportBody = (request: Soda3ExportRequest) => ({
|
|
56
|
+
query: request.query,
|
|
57
|
+
parameters: request.parameters,
|
|
58
|
+
timeout: request.timeout,
|
|
59
|
+
serializationOptions: request.serializationOptions
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
export const rangeHeader = (request: Soda3ExportRequest) =>
|
|
63
|
+
request.range === undefined
|
|
64
|
+
? undefined
|
|
65
|
+
: `bytes=${request.range.start}-${request.range.end ?? ""}`
|
|
66
|
+
|
|
67
|
+
export const mapHttpClientError = (operation: string, error: HttpClientError.HttpClientError) => {
|
|
68
|
+
const info = httpFailureInfo(error)
|
|
69
|
+
|
|
70
|
+
return ProviderHttpError.make({
|
|
71
|
+
operation,
|
|
72
|
+
status: info.status,
|
|
73
|
+
statusText: info.statusText,
|
|
74
|
+
retryable: info.status === 0 || retryableStatus(info.status)
|
|
75
|
+
})
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export const decodeJson = (operation: string, response: HttpClientResponse.HttpClientResponse) =>
|
|
79
|
+
response.json.pipe(
|
|
80
|
+
Effect.catchTag("HttpClientError", () =>
|
|
81
|
+
Effect.fail(
|
|
82
|
+
ProviderContractError.make({
|
|
83
|
+
operation,
|
|
84
|
+
message: "Provider response was not valid JSON"
|
|
85
|
+
})
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
export const decodeQueryResponse = (input: unknown) =>
|
|
91
|
+
decodeUnknown(Soda3QueryResponse, input).pipe(
|
|
92
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
93
|
+
decodeUnknown(providerRows, input).pipe(
|
|
94
|
+
Effect.map((rows) =>
|
|
95
|
+
Soda3QueryResponse.make({
|
|
96
|
+
rows
|
|
97
|
+
})
|
|
34
98
|
),
|
|
35
|
-
|
|
36
|
-
|
|
99
|
+
Effect.catchTag("SchemaError", () =>
|
|
100
|
+
Effect.fail(
|
|
101
|
+
ProviderContractError.make({
|
|
102
|
+
operation: "query",
|
|
103
|
+
message: error.message
|
|
104
|
+
})
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
export const decodeCatalogSearchResponse = (input: unknown) =>
|
|
112
|
+
decodeUnknown(Soda3CatalogSearchResponse, input).pipe(
|
|
113
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
114
|
+
Effect.fail(
|
|
115
|
+
ProviderContractError.make({
|
|
116
|
+
operation: "catalog",
|
|
117
|
+
message: error.message
|
|
118
|
+
})
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
)
|
|
37
122
|
|
|
38
123
|
export const requestWithPolicy = <A>(
|
|
39
124
|
operation: string,
|
|
@@ -41,52 +126,29 @@ export const requestWithPolicy = <A>(
|
|
|
41
126
|
) =>
|
|
42
127
|
Effect.gen(function* () {
|
|
43
128
|
const config = yield* Soda3ClientConfig
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
)
|
|
59
|
-
: effect
|
|
60
|
-
|
|
61
|
-
const timeoutMs = config.timeoutMs
|
|
62
|
-
const timed =
|
|
63
|
-
timeoutMs === undefined
|
|
64
|
-
? retried
|
|
65
|
-
: retried.pipe(
|
|
66
|
-
Effect.timeoutOrElse({
|
|
67
|
-
duration: timeoutMs,
|
|
68
|
-
orElse: () =>
|
|
69
|
-
Effect.fail(
|
|
70
|
-
TimeoutError.make({
|
|
71
|
-
operation,
|
|
72
|
-
timeoutMs
|
|
73
|
-
})
|
|
74
|
-
)
|
|
75
|
-
})
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
return yield* timed
|
|
129
|
+
return yield* withRequestPolicy(config, effect, {
|
|
130
|
+
isRetryable: isRetryableProviderError,
|
|
131
|
+
onExhausted: (attempts, error) =>
|
|
132
|
+
RetryExhaustedError.make({
|
|
133
|
+
operation,
|
|
134
|
+
attempts,
|
|
135
|
+
message: `Retry attempts exhausted after provider status ${error.status}`
|
|
136
|
+
}),
|
|
137
|
+
onTimeout: (timeoutMs) =>
|
|
138
|
+
TimeoutError.make({
|
|
139
|
+
operation,
|
|
140
|
+
timeoutMs
|
|
141
|
+
})
|
|
142
|
+
})
|
|
79
143
|
})
|
|
80
144
|
|
|
81
|
-
const httpStatusText = (status: number) => `HTTP ${status}`
|
|
82
|
-
|
|
83
145
|
export const executeRequest = (operation: string, request: HttpClientRequest.HttpClientRequest) =>
|
|
84
146
|
Effect.gen(function* () {
|
|
85
147
|
const response = yield* HttpClient.execute(request).pipe(
|
|
86
148
|
Effect.mapError((error) => mapHttpClientError(operation, error))
|
|
87
149
|
)
|
|
88
150
|
|
|
89
|
-
if (response.status
|
|
151
|
+
if (!isOkStatus(response.status)) {
|
|
90
152
|
return yield* ProviderHttpError.make({
|
|
91
153
|
operation,
|
|
92
154
|
status: response.status,
|
package/src/metadata.ts
CHANGED
|
@@ -4,23 +4,13 @@ import {
|
|
|
4
4
|
type Soda3DatasetMetadataRequestInput
|
|
5
5
|
} from "@nyc-transit-kit/contracts/soda3"
|
|
6
6
|
import * as Effect from "effect/Effect"
|
|
7
|
-
import * as Schema from "effect/Schema"
|
|
8
7
|
import { catalogSearch } from "./catalog"
|
|
9
|
-
import {
|
|
8
|
+
import { ProviderContractError } from "./errors"
|
|
9
|
+
import { decodeInput } from "./internal/decode"
|
|
10
10
|
|
|
11
11
|
export type { Soda3DatasetMetadataRequestInput }
|
|
12
12
|
|
|
13
|
-
const decodeDatasetMetadataRequest = (
|
|
14
|
-
Schema.decodeUnknownEffect(Soda3DatasetMetadataRequest)(input).pipe(
|
|
15
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
16
|
-
Effect.fail(
|
|
17
|
-
InvalidInputError.make({
|
|
18
|
-
operation: "metadata",
|
|
19
|
-
message: error.message
|
|
20
|
-
})
|
|
21
|
-
)
|
|
22
|
-
)
|
|
23
|
-
)
|
|
13
|
+
const decodeDatasetMetadataRequest = decodeInput("metadata", Soda3DatasetMetadataRequest)
|
|
24
14
|
|
|
25
15
|
const matchingResult = (results: ReadonlyArray<Soda3CatalogResultFragment>, datasetId: string) =>
|
|
26
16
|
results.find((result) => String(result.resource.id) === datasetId)
|
package/src/query.ts
CHANGED
|
@@ -4,14 +4,19 @@ import {
|
|
|
4
4
|
type Soda3Row
|
|
5
5
|
} from "@nyc-transit-kit/contracts/soda3"
|
|
6
6
|
import * as Effect from "effect/Effect"
|
|
7
|
-
import * as Schema from "effect/Schema"
|
|
8
7
|
import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
8
|
+
import { Soda3ClientConfig } from "./client"
|
|
9
9
|
import { buildQueryUrl, decodeSoda3QueryRequest } from "./endpoints"
|
|
10
10
|
import { InvalidInputError, ProviderContractError } from "./errors"
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
import { decodeInput } from "./internal/decode"
|
|
12
|
+
import {
|
|
13
|
+
addHeaders,
|
|
14
|
+
decodeJson,
|
|
15
|
+
decodeQueryResponse,
|
|
16
|
+
executeRequest,
|
|
17
|
+
requestWithPolicy,
|
|
18
|
+
toQueryBody
|
|
19
|
+
} from "./internal/transport"
|
|
15
20
|
|
|
16
21
|
export type { Soda3QueryRequestInput }
|
|
17
22
|
|
|
@@ -28,14 +33,15 @@ export type Soda3CountRowsInput = {
|
|
|
28
33
|
}
|
|
29
34
|
|
|
30
35
|
const decodeHelperPositiveInteger = (name: string, value: number) =>
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
decodeInput(
|
|
37
|
+
"query",
|
|
38
|
+
PositiveInteger
|
|
39
|
+
)(value).pipe(
|
|
40
|
+
Effect.mapError((error) =>
|
|
41
|
+
InvalidInputError.make({
|
|
42
|
+
operation: "query",
|
|
43
|
+
message: `Invalid ${name}: ${error.message}`
|
|
44
|
+
})
|
|
39
45
|
)
|
|
40
46
|
)
|
|
41
47
|
|
package/src/soql.ts
CHANGED
|
@@ -3,351 +3,212 @@ import { PositiveInteger } from "@nyc-transit-kit/contracts/soda3"
|
|
|
3
3
|
import * as Effect from "effect/Effect"
|
|
4
4
|
import * as Schema from "effect/Schema"
|
|
5
5
|
import { InvalidInputError } from "./errors"
|
|
6
|
-
|
|
6
|
+
import { decodeInput } from "./internal/decode"
|
|
7
7
|
export const soqlSelectAll = "SELECT *"
|
|
8
|
-
|
|
9
8
|
export type SoqlParameters = Readonly<Record<string, unknown>>
|
|
10
|
-
|
|
11
|
-
export type
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
readonly year: number
|
|
23
|
-
readonly month: number
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
const soqlIdentifierPattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
27
|
-
const soqlParameterNamePattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
28
|
-
const isoDatePartsPattern = /^(\d{4})-(\d{2})-(\d{2})$/
|
|
29
|
-
|
|
30
|
-
const SoqlIdentifier = Schema.String.pipe(
|
|
31
|
-
Schema.check(
|
|
32
|
-
Schema.isPattern(soqlIdentifierPattern, {
|
|
33
|
-
message: "Expected a SoQL identifier such as route_id"
|
|
34
|
-
})
|
|
35
|
-
)
|
|
9
|
+
export type SoqlFragment = Readonly<{ text: string; parameters?: SoqlParameters }>
|
|
10
|
+
export type SocrataTimestampWindow = Readonly<{ start: string; end: string }>
|
|
11
|
+
export type SocrataYearMonth = Readonly<{ year: number; month: number }>
|
|
12
|
+
const fn = Effect.fn
|
|
13
|
+
const identifierPattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
14
|
+
const parameterPattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
15
|
+
const datePattern = /^(\d{4})-(\d{2})-(\d{2})$/
|
|
16
|
+
const patternString = (pattern: RegExp, message: string) =>
|
|
17
|
+
Schema.String.pipe(Schema.check(Schema.isPattern(pattern, { message })))
|
|
18
|
+
const SoqlIdentifier = patternString(
|
|
19
|
+
identifierPattern,
|
|
20
|
+
"Expected a SoQL identifier such as route_id"
|
|
36
21
|
)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
Schema.isPattern(soqlParameterNamePattern, {
|
|
41
|
-
message: "Expected a SoQL parameter name such as route"
|
|
42
|
-
})
|
|
43
|
-
)
|
|
22
|
+
const SoqlParameterName = patternString(
|
|
23
|
+
parameterPattern,
|
|
24
|
+
"Expected a SoQL parameter name such as route"
|
|
44
25
|
)
|
|
45
|
-
|
|
46
26
|
const SoqlSortDirection = Schema.Literals(["ASC", "DESC"])
|
|
47
27
|
export type SoqlSortDirection = typeof SoqlSortDirection.Type
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
)
|
|
53
|
-
|
|
54
|
-
const SocrataMonth =
|
|
55
|
-
|
|
56
|
-
Schema.check(Schema.isLessThanOrEqualTo(12))
|
|
57
|
-
)
|
|
58
|
-
|
|
59
|
-
const SocrataYearMonthSchema = Schema.Struct({
|
|
60
|
-
year: SocrataYear,
|
|
61
|
-
month: SocrataMonth
|
|
62
|
-
})
|
|
63
|
-
|
|
28
|
+
const intRange = (minimum: number, maximum: number) =>
|
|
29
|
+
Schema.Int.pipe(
|
|
30
|
+
Schema.check(Schema.isGreaterThanOrEqualTo(minimum)),
|
|
31
|
+
Schema.check(Schema.isLessThanOrEqualTo(maximum))
|
|
32
|
+
)
|
|
33
|
+
const SocrataYear = intRange(1900, 9999)
|
|
34
|
+
const SocrataMonth = intRange(1, 12)
|
|
35
|
+
const SocrataYearMonthSchema = Schema.Struct({ year: SocrataYear, month: SocrataMonth })
|
|
64
36
|
const invalidQueryInput = (message: string) =>
|
|
65
|
-
InvalidInputError.make({
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
const
|
|
79
|
-
const
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const decodeMonth = decodeQueryInput(SocrataMonth)
|
|
84
|
-
const decodeYearMonth = decodeQueryInput(SocrataYearMonthSchema)
|
|
85
|
-
|
|
86
|
-
const withParameters = (
|
|
87
|
-
text: string,
|
|
88
|
-
parameters: Readonly<Record<string, unknown>>
|
|
89
|
-
): SoqlFragment => ({
|
|
90
|
-
text,
|
|
91
|
-
parameters
|
|
92
|
-
})
|
|
93
|
-
|
|
94
|
-
const withoutParameters = (text: string): SoqlFragment => ({
|
|
95
|
-
text
|
|
96
|
-
})
|
|
97
|
-
|
|
98
|
-
const mergeParameters = (fragments: ReadonlyArray<SoqlFragment>) =>
|
|
99
|
-
Effect.gen(function* () {
|
|
100
|
-
const parameters: Record<string, unknown> = {}
|
|
101
|
-
|
|
102
|
-
for (const fragment of fragments) {
|
|
103
|
-
for (const [key, value] of Object.entries(fragment.parameters ?? {})) {
|
|
104
|
-
if (Object.hasOwn(parameters, key)) {
|
|
105
|
-
return yield* invalidQueryInput(`Duplicate SoQL parameter name: ${key}`)
|
|
106
|
-
}
|
|
107
|
-
parameters[key] = value
|
|
37
|
+
InvalidInputError.make({ operation: "query", message })
|
|
38
|
+
const decode = <S extends Schema.Top>(schema: S) => decodeInput("query", schema)
|
|
39
|
+
const decodeLimit = decode(PositiveInteger)
|
|
40
|
+
const decodeIdentifier = decode(SoqlIdentifier)
|
|
41
|
+
const decodeParameterName = decode(SoqlParameterName)
|
|
42
|
+
const decodeSortDirection = decode(SoqlSortDirection)
|
|
43
|
+
const decodeIsoDate = decode(IsoDate)
|
|
44
|
+
const decodeYear = decode(SocrataYear)
|
|
45
|
+
const decodeMonth = decode(SocrataMonth)
|
|
46
|
+
const decodeYearMonth = decode(SocrataYearMonthSchema)
|
|
47
|
+
const fragment = (text: string, parameters?: SoqlParameters): SoqlFragment =>
|
|
48
|
+
parameters === undefined ? { text } : { text, parameters }
|
|
49
|
+
const mergeParameters = (fragments: ReadonlyArray<SoqlFragment>) => {
|
|
50
|
+
const parameters: Record<string, unknown> = {}
|
|
51
|
+
for (const part of fragments) {
|
|
52
|
+
for (const [key, value] of Object.entries(part.parameters ?? {})) {
|
|
53
|
+
if (Object.hasOwn(parameters, key)) {
|
|
54
|
+
return Effect.fail(invalidQueryInput(`Duplicate SoQL parameter name: ${key}`))
|
|
108
55
|
}
|
|
56
|
+
parameters[key] = value
|
|
109
57
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
const
|
|
115
|
-
|
|
58
|
+
}
|
|
59
|
+
return Effect.succeed(Object.keys(parameters).length === 0 ? undefined : parameters)
|
|
60
|
+
}
|
|
61
|
+
const dateMessage = "Expected an ISO calendar date in YYYY-MM-DD format"
|
|
62
|
+
const pad = (value: number, length: number) => value.toString().padStart(length, "0")
|
|
116
63
|
const timestampFromParts = (year: number, month: number, day: number) =>
|
|
117
|
-
`${year
|
|
118
|
-
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const yearText = match[1]
|
|
127
|
-
const monthText = match[2]
|
|
128
|
-
const dayText = match[3]
|
|
129
|
-
if (yearText === undefined || monthText === undefined || dayText === undefined) {
|
|
130
|
-
return yield* invalidQueryInput("Expected an ISO calendar date in YYYY-MM-DD format")
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
return {
|
|
134
|
-
year: Number(yearText),
|
|
135
|
-
month: Number(monthText),
|
|
136
|
-
day: Number(dayText)
|
|
137
|
-
}
|
|
138
|
-
})
|
|
139
|
-
|
|
140
|
-
const utcTimestampFromDate = (date: Date) =>
|
|
141
|
-
Effect.gen(function* () {
|
|
142
|
-
if (Number.isNaN(date.getTime())) {
|
|
143
|
-
return yield* invalidQueryInput("Expected a valid Date")
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
return timestampFromParts(date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate())
|
|
147
|
-
})
|
|
148
|
-
|
|
149
|
-
const epochDay = (value: string) =>
|
|
150
|
-
Effect.gen(function* () {
|
|
151
|
-
const parts = yield* dateParts(value)
|
|
152
|
-
return Date.UTC(parts.year, parts.month - 1, parts.day)
|
|
153
|
-
})
|
|
154
|
-
|
|
155
|
-
export const soqlLimit = Effect.fn("Soda3.soqlLimit")(function* (query: string, limit: unknown) {
|
|
156
|
-
const decodedLimit = yield* decodeLimit(limit)
|
|
157
|
-
return `${query} LIMIT ${decodedLimit}`
|
|
64
|
+
`${pad(year, 4)}-${pad(month, 2)}-${pad(day, 2)}T00:00:00`
|
|
65
|
+
const datePartsOf = (value: string) => {
|
|
66
|
+
const [, yearText, monthText, dayText] = datePattern.exec(value) ?? []
|
|
67
|
+
return yearText === undefined || monthText === undefined || dayText === undefined
|
|
68
|
+
? Effect.fail(invalidQueryInput(dateMessage))
|
|
69
|
+
: Effect.succeed({ year: +yearText, month: +monthText, day: +dayText })
|
|
70
|
+
}
|
|
71
|
+
export const soqlLimit = fn("Soda3.soqlLimit")(function* (query: string, limit: unknown) {
|
|
72
|
+
return `${query} LIMIT ${yield* decodeLimit(limit)}`
|
|
158
73
|
})
|
|
159
|
-
|
|
160
|
-
export const soqlIdentifier = Effect.fn("Soda3.soqlIdentifier")(function* (input: unknown) {
|
|
74
|
+
export const soqlIdentifier = fn("Soda3.soqlIdentifier")(function* (input: unknown) {
|
|
161
75
|
return yield* decodeIdentifier(input)
|
|
162
76
|
})
|
|
163
|
-
|
|
164
|
-
export const soqlParameterName = Effect.fn("Soda3.soqlParameterName")(function* (input: unknown) {
|
|
77
|
+
export const soqlParameterName = fn("Soda3.soqlParameterName")(function* (input: unknown) {
|
|
165
78
|
return yield* decodeParameterName(input)
|
|
166
79
|
})
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
value: unknown
|
|
171
|
-
) {
|
|
172
|
-
const decodedName = yield* decodeParameterName(name)
|
|
173
|
-
return withParameters(`:${decodedName}`, {
|
|
174
|
-
[decodedName]: value
|
|
175
|
-
})
|
|
80
|
+
export const soqlParameter = fn("Soda3.soqlParameter")(function* (name: unknown, value: unknown) {
|
|
81
|
+
const key = yield* decodeParameterName(name)
|
|
82
|
+
return fragment(`:${key}`, { [key]: value })
|
|
176
83
|
})
|
|
177
|
-
|
|
178
|
-
export const soqlEq = Effect.fn("Soda3.soqlEq")(function* (
|
|
84
|
+
export const soqlEq = fn("Soda3.soqlEq")(function* (
|
|
179
85
|
column: unknown,
|
|
180
86
|
parameterName: unknown,
|
|
181
87
|
value: unknown
|
|
182
88
|
) {
|
|
183
|
-
const
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
return withParameters(`${decodedColumn} = :${decodedParameterName}`, {
|
|
187
|
-
[decodedParameterName]: value
|
|
188
|
-
})
|
|
89
|
+
const col = yield* decodeIdentifier(column)
|
|
90
|
+
const key = yield* decodeParameterName(parameterName)
|
|
91
|
+
return fragment(`${col} = :${key}`, { [key]: value })
|
|
189
92
|
})
|
|
190
|
-
|
|
191
|
-
export const soqlIn = Effect.fn("Soda3.soqlIn")(function* (
|
|
93
|
+
export const soqlIn = fn("Soda3.soqlIn")(function* (
|
|
192
94
|
column: unknown,
|
|
193
95
|
parameterBaseName: unknown,
|
|
194
96
|
values: ReadonlyArray<unknown>
|
|
195
97
|
) {
|
|
196
|
-
const decodedColumn = yield* decodeIdentifier(column)
|
|
197
|
-
const decodedParameterBaseName = yield* decodeParameterName(parameterBaseName)
|
|
198
|
-
|
|
199
98
|
if (values.length === 0) {
|
|
200
99
|
return yield* invalidQueryInput("SoQL IN predicates require at least one value")
|
|
201
100
|
}
|
|
202
|
-
|
|
101
|
+
const col = yield* decodeIdentifier(column)
|
|
102
|
+
const base = yield* decodeParameterName(parameterBaseName)
|
|
203
103
|
const parameters: Record<string, unknown> = {}
|
|
204
104
|
const placeholders: Array<string> = []
|
|
205
|
-
|
|
206
105
|
for (let index = 0; index < values.length; index += 1) {
|
|
207
|
-
const
|
|
208
|
-
parameters[
|
|
209
|
-
placeholders.push(`:${
|
|
106
|
+
const key = `${base}_${index + 1}`
|
|
107
|
+
parameters[key] = values[index]
|
|
108
|
+
placeholders.push(`:${key}`)
|
|
210
109
|
}
|
|
211
|
-
|
|
212
|
-
return withParameters(`${decodedColumn} IN (${placeholders.join(", ")})`, parameters)
|
|
110
|
+
return fragment(`${col} IN (${placeholders.join(", ")})`, parameters)
|
|
213
111
|
})
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const decodedColumn = yield* decodeIdentifier(column)
|
|
217
|
-
return withoutParameters(`${decodedColumn} IS NOT NULL`)
|
|
112
|
+
export const soqlIsNotNull = fn("Soda3.soqlIsNotNull")(function* (column: unknown) {
|
|
113
|
+
return fragment(`${yield* decodeIdentifier(column)} IS NOT NULL`)
|
|
218
114
|
})
|
|
219
|
-
|
|
220
|
-
export const soqlAnd = Effect.fn("Soda3.soqlAnd")(function* (
|
|
221
|
-
fragments: ReadonlyArray<SoqlFragment>
|
|
222
|
-
) {
|
|
115
|
+
export const soqlAnd = fn("Soda3.soqlAnd")(function* (fragments: ReadonlyArray<SoqlFragment>) {
|
|
223
116
|
if (fragments.length === 0) {
|
|
224
117
|
return yield* invalidQueryInput("SoQL AND composition requires at least one fragment")
|
|
225
118
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
return parameters === undefined ? withoutParameters(text) : withParameters(text, parameters)
|
|
119
|
+
return fragment(
|
|
120
|
+
fragments.map((part) => `(${part.text})`).join(" AND "),
|
|
121
|
+
yield* mergeParameters(fragments)
|
|
122
|
+
)
|
|
231
123
|
})
|
|
232
|
-
|
|
233
|
-
export const soqlOrderBy = Effect.fn("Soda3.soqlOrderBy")(function* (
|
|
124
|
+
export const soqlOrderBy = fn("Soda3.soqlOrderBy")(function* (
|
|
234
125
|
column: unknown,
|
|
235
126
|
direction: unknown = "ASC"
|
|
236
127
|
) {
|
|
237
|
-
const
|
|
238
|
-
|
|
239
|
-
return withoutParameters(`ORDER BY ${decodedColumn} ${decodedDirection}`)
|
|
128
|
+
const col = yield* decodeIdentifier(column)
|
|
129
|
+
return fragment(`ORDER BY ${col} ${yield* decodeSortDirection(direction)}`)
|
|
240
130
|
})
|
|
241
|
-
|
|
242
|
-
export const socrataTimestamp = Effect.fn("Soda3.socrataTimestamp")(function* (
|
|
243
|
-
date: string | Date
|
|
244
|
-
) {
|
|
131
|
+
export const socrataTimestamp = fn("Soda3.socrataTimestamp")(function* (date: string | Date) {
|
|
245
132
|
if (date instanceof Date) {
|
|
246
|
-
|
|
133
|
+
if (Number.isNaN(date.getTime())) {
|
|
134
|
+
return yield* invalidQueryInput("Expected a valid Date")
|
|
135
|
+
}
|
|
136
|
+
return timestampFromParts(date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate())
|
|
247
137
|
}
|
|
248
|
-
|
|
249
138
|
const decodedDate = yield* decodeIsoDate(date)
|
|
250
|
-
const parts = yield*
|
|
139
|
+
const parts = yield* datePartsOf(decodedDate)
|
|
251
140
|
return timestampFromParts(parts.year, parts.month, parts.day)
|
|
252
141
|
})
|
|
253
|
-
|
|
254
|
-
export const socrataMonthWindow = Effect.fn("Soda3.socrataMonthWindow")(function* (
|
|
142
|
+
export const socrataMonthWindow = fn("Soda3.socrataMonthWindow")(function* (
|
|
255
143
|
year: unknown,
|
|
256
144
|
month: unknown
|
|
257
145
|
) {
|
|
258
|
-
const
|
|
259
|
-
const
|
|
260
|
-
const endYear =
|
|
261
|
-
const endMonth =
|
|
262
|
-
|
|
146
|
+
const y = yield* decodeYear(year)
|
|
147
|
+
const m = yield* decodeMonth(month)
|
|
148
|
+
const endYear = m === 12 ? y + 1 : y
|
|
149
|
+
const endMonth = m === 12 ? 1 : m + 1
|
|
263
150
|
return {
|
|
264
|
-
start: timestampFromParts(
|
|
151
|
+
start: timestampFromParts(y, m, 1),
|
|
265
152
|
end: timestampFromParts(endYear, endMonth, 1)
|
|
266
153
|
}
|
|
267
154
|
})
|
|
268
|
-
|
|
269
|
-
export const socrataDateWindow = Effect.fn("Soda3.socrataDateWindow")(function* (
|
|
155
|
+
export const socrataDateWindow = fn("Soda3.socrataDateWindow")(function* (
|
|
270
156
|
startDate: unknown,
|
|
271
157
|
endDate: unknown
|
|
272
158
|
) {
|
|
273
|
-
const
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
if (startEpochDay >= endEpochDay) {
|
|
159
|
+
const start = yield* datePartsOf(yield* decodeIsoDate(startDate))
|
|
160
|
+
const end = yield* datePartsOf(yield* decodeIsoDate(endDate))
|
|
161
|
+
if (
|
|
162
|
+
Date.UTC(start.year, start.month - 1, start.day) >= Date.UTC(end.year, end.month - 1, end.day)
|
|
163
|
+
) {
|
|
279
164
|
return yield* invalidQueryInput("Socrata date windows require startDate before endDate")
|
|
280
165
|
}
|
|
281
|
-
|
|
282
166
|
return {
|
|
283
|
-
start:
|
|
284
|
-
end:
|
|
167
|
+
start: timestampFromParts(start.year, start.month, start.day),
|
|
168
|
+
end: timestampFromParts(end.year, end.month, end.day)
|
|
285
169
|
}
|
|
286
170
|
})
|
|
287
|
-
|
|
288
|
-
export const soqlTimestampRange = Effect.fn("Soda3.soqlTimestampRange")(function* (
|
|
171
|
+
export const soqlTimestampRange = fn("Soda3.soqlTimestampRange")(function* (
|
|
289
172
|
column: unknown,
|
|
290
173
|
startParameterName: unknown,
|
|
291
174
|
endParameterName: unknown,
|
|
292
|
-
|
|
175
|
+
w: SocrataTimestampWindow
|
|
293
176
|
) {
|
|
294
|
-
const
|
|
295
|
-
const
|
|
296
|
-
const
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
return yield* invalidQueryInput(`Duplicate SoQL parameter name: ${decodedStartParameterName}`)
|
|
177
|
+
const col = yield* decodeIdentifier(column)
|
|
178
|
+
const s = yield* decodeParameterName(startParameterName)
|
|
179
|
+
const e = yield* decodeParameterName(endParameterName)
|
|
180
|
+
if (s === e) {
|
|
181
|
+
return yield* invalidQueryInput(`Duplicate SoQL parameter name: ${s}`)
|
|
300
182
|
}
|
|
301
|
-
|
|
302
|
-
return withParameters(
|
|
303
|
-
`${decodedColumn} >= :${decodedStartParameterName} AND ${decodedColumn} < :${decodedEndParameterName}`,
|
|
304
|
-
{
|
|
305
|
-
[decodedStartParameterName]: window.start,
|
|
306
|
-
[decodedEndParameterName]: window.end
|
|
307
|
-
}
|
|
308
|
-
)
|
|
183
|
+
return fragment(`${col} >= :${s} AND ${col} < :${e}`, { [s]: w.start, [e]: w.end })
|
|
309
184
|
})
|
|
310
|
-
|
|
311
|
-
export const soqlMonthWindow = Effect.fn("Soda3.soqlMonthWindow")(function* (
|
|
185
|
+
export const soqlMonthWindow = fn("Soda3.soqlMonthWindow")(function* (
|
|
312
186
|
column: unknown,
|
|
313
187
|
year: unknown,
|
|
314
188
|
month: unknown,
|
|
315
189
|
parameterBaseName: unknown
|
|
316
190
|
) {
|
|
317
|
-
const
|
|
191
|
+
const base = yield* decodeParameterName(parameterBaseName)
|
|
318
192
|
const window = yield* socrataMonthWindow(year, month)
|
|
319
|
-
return yield* soqlTimestampRange(
|
|
320
|
-
column,
|
|
321
|
-
`${decodedParameterBaseName}_start`,
|
|
322
|
-
`${decodedParameterBaseName}_end`,
|
|
323
|
-
window
|
|
324
|
-
)
|
|
193
|
+
return yield* soqlTimestampRange(column, `${base}_start`, `${base}_end`, window)
|
|
325
194
|
})
|
|
326
|
-
|
|
327
|
-
export const soqlYearMonthRange = Effect.fn("Soda3.soqlYearMonthRange")(function* (
|
|
195
|
+
export const soqlYearMonthRange = fn("Soda3.soqlYearMonthRange")(function* (
|
|
328
196
|
yearColumn: unknown,
|
|
329
197
|
monthColumn: unknown,
|
|
330
198
|
start: unknown,
|
|
331
199
|
end: unknown
|
|
332
200
|
) {
|
|
333
|
-
const
|
|
334
|
-
const
|
|
335
|
-
const
|
|
336
|
-
const
|
|
337
|
-
const startOrder =
|
|
338
|
-
const endOrder =
|
|
339
|
-
|
|
201
|
+
const yearCol = yield* decodeIdentifier(yearColumn)
|
|
202
|
+
const monthCol = yield* decodeIdentifier(monthColumn)
|
|
203
|
+
const rs = yield* decodeYearMonth(start)
|
|
204
|
+
const re = yield* decodeYearMonth(end)
|
|
205
|
+
const startOrder = rs.year * 12 + rs.month
|
|
206
|
+
const endOrder = re.year * 12 + re.month
|
|
340
207
|
if (startOrder > endOrder) {
|
|
341
208
|
return yield* invalidQueryInput("SoQL year/month ranges require start before end")
|
|
342
209
|
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
{
|
|
347
|
-
start_year: decodedStart.year,
|
|
348
|
-
start_month: decodedStart.month,
|
|
349
|
-
end_year: decodedEnd.year,
|
|
350
|
-
end_month: decodedEnd.month
|
|
351
|
-
}
|
|
210
|
+
return fragment(
|
|
211
|
+
`(${yearCol} > :start_year OR (${yearCol} = :start_year AND ${monthCol} >= :start_month)) AND (${yearCol} < :end_year OR (${yearCol} = :end_year AND ${monthCol} <= :end_month))`,
|
|
212
|
+
{ start_year: rs.year, start_month: rs.month, end_year: re.year, end_month: re.month }
|
|
352
213
|
)
|
|
353
214
|
})
|
package/src/transport.ts
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import * as Duration from "effect/Duration"
|
|
2
|
+
import * as Effect from "effect/Effect"
|
|
3
|
+
import * as Layer from "effect/Layer"
|
|
4
|
+
import * as Schedule from "effect/Schedule"
|
|
5
|
+
import * as Stream from "effect/Stream"
|
|
6
|
+
import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
|
|
7
|
+
import type * as Headers from "effect/unstable/http/Headers"
|
|
8
|
+
import type * as HttpClientError from "effect/unstable/http/HttpClientError"
|
|
9
|
+
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
10
|
+
|
|
11
|
+
export type FetchImplementation = (
|
|
12
|
+
input: string | URL | globalThis.Request,
|
|
13
|
+
init?: globalThis.RequestInit
|
|
14
|
+
) => Promise<globalThis.Response>
|
|
15
|
+
|
|
16
|
+
export interface RequestPolicyConfig {
|
|
17
|
+
readonly retryTimes?: number
|
|
18
|
+
readonly retryDelayMs?: number
|
|
19
|
+
readonly timeoutMs?: number
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export const httpStatusText = (status: number) => `HTTP ${status}`
|
|
23
|
+
export const retryableStatus = (status: number) => status === 429 || status >= 500
|
|
24
|
+
export const isOkStatus = (status: number) => status >= 200 && status < 300
|
|
25
|
+
|
|
26
|
+
const retryPolicy = <E>(
|
|
27
|
+
retryTimes: number,
|
|
28
|
+
retryDelayMs: number | undefined,
|
|
29
|
+
isRetryable: (error: E) => boolean
|
|
30
|
+
) =>
|
|
31
|
+
retryDelayMs === undefined || retryDelayMs <= 0
|
|
32
|
+
? {
|
|
33
|
+
times: retryTimes,
|
|
34
|
+
while: isRetryable
|
|
35
|
+
}
|
|
36
|
+
: {
|
|
37
|
+
schedule: Schedule.exponential(Duration.millis(retryDelayMs)).pipe(
|
|
38
|
+
Schedule.both(Schedule.recurs(retryTimes))
|
|
39
|
+
),
|
|
40
|
+
while: isRetryable
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export const withRequestPolicy = <A, E, R, Retryable extends E>(
|
|
44
|
+
config: RequestPolicyConfig,
|
|
45
|
+
effect: Effect.Effect<A, E, R>,
|
|
46
|
+
adapters: {
|
|
47
|
+
readonly isRetryable: (error: E) => error is Retryable
|
|
48
|
+
readonly onExhausted: (attempts: number, error: Retryable) => E
|
|
49
|
+
readonly onTimeout: (timeoutMs: number) => E
|
|
50
|
+
readonly retryableResult?: (value: A) => E | undefined
|
|
51
|
+
}
|
|
52
|
+
): Effect.Effect<A, E, R> => {
|
|
53
|
+
const retryTimes = config.retryTimes ?? 0
|
|
54
|
+
const effectWithRetryableResults =
|
|
55
|
+
retryTimes > 0 && adapters.retryableResult !== undefined
|
|
56
|
+
? effect.pipe(
|
|
57
|
+
Effect.flatMap((value) => {
|
|
58
|
+
const retryableError = adapters.retryableResult?.(value)
|
|
59
|
+
return retryableError === undefined
|
|
60
|
+
? Effect.succeed(value)
|
|
61
|
+
: Effect.fail(retryableError)
|
|
62
|
+
})
|
|
63
|
+
)
|
|
64
|
+
: effect
|
|
65
|
+
const retried =
|
|
66
|
+
retryTimes > 0
|
|
67
|
+
? effectWithRetryableResults.pipe(
|
|
68
|
+
Effect.retry(retryPolicy(retryTimes, config.retryDelayMs, adapters.isRetryable)),
|
|
69
|
+
Effect.catchIf(adapters.isRetryable, (error) =>
|
|
70
|
+
Effect.fail(adapters.onExhausted(retryTimes + 1, error))
|
|
71
|
+
)
|
|
72
|
+
)
|
|
73
|
+
: effectWithRetryableResults
|
|
74
|
+
const timeoutMs = config.timeoutMs
|
|
75
|
+
|
|
76
|
+
return timeoutMs === undefined
|
|
77
|
+
? retried
|
|
78
|
+
: retried.pipe(
|
|
79
|
+
Effect.timeoutOrElse({
|
|
80
|
+
duration: timeoutMs,
|
|
81
|
+
orElse: () => Effect.fail(adapters.onTimeout(timeoutMs))
|
|
82
|
+
})
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export const httpFailureInfo = (error: HttpClientError.HttpClientError) => {
|
|
87
|
+
const response = error.response
|
|
88
|
+
|
|
89
|
+
return response === undefined
|
|
90
|
+
? {
|
|
91
|
+
status: 0,
|
|
92
|
+
statusText: error.message
|
|
93
|
+
}
|
|
94
|
+
: {
|
|
95
|
+
status: response.status,
|
|
96
|
+
statusText: httpStatusText(response.status)
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export const toWebResponse = (response: HttpClientResponse.HttpClientResponse) =>
|
|
101
|
+
Effect.map(
|
|
102
|
+
Stream.toReadableStreamEffect(response.stream),
|
|
103
|
+
(body) =>
|
|
104
|
+
new Response(body, {
|
|
105
|
+
status: response.status,
|
|
106
|
+
headers: response.headers
|
|
107
|
+
})
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
export const contentLengthOf = (headers: Headers.Headers) => {
|
|
111
|
+
const value = headers["content-length"]
|
|
112
|
+
if (value === undefined) {
|
|
113
|
+
return undefined
|
|
114
|
+
}
|
|
115
|
+
const parsed = Number(value)
|
|
116
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const shouldRedactSearchParameter = (name: string) => {
|
|
120
|
+
const lower = name.toLowerCase()
|
|
121
|
+
return lower.includes("key") || lower.includes("token") || lower.includes("secret")
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export const redactedUrl = (url: string) => {
|
|
125
|
+
try {
|
|
126
|
+
const parsed = new URL(url)
|
|
127
|
+
|
|
128
|
+
for (const name of parsed.searchParams.keys()) {
|
|
129
|
+
if (shouldRedactSearchParameter(name)) {
|
|
130
|
+
parsed.searchParams.set(name, "REDACTED")
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return parsed.toString()
|
|
135
|
+
} catch {
|
|
136
|
+
return "[invalid-url]"
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export const fetchHttpClientLayer = (fetchImplementation?: FetchImplementation) =>
|
|
141
|
+
fetchImplementation === undefined
|
|
142
|
+
? FetchHttpClient.layer
|
|
143
|
+
: FetchHttpClient.layer.pipe(
|
|
144
|
+
Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fetchImplementation as typeof fetch))
|
|
145
|
+
)
|
package/src/internal/bodies.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { Soda3ExportRequest, Soda3QueryRequest } from "@nyc-transit-kit/contracts/soda3"
|
|
2
|
-
|
|
3
|
-
export const toQueryBody = (request: Soda3QueryRequest) => ({
|
|
4
|
-
query: request.query,
|
|
5
|
-
page: request.page,
|
|
6
|
-
parameters: request.parameters,
|
|
7
|
-
timeout: request.timeout,
|
|
8
|
-
includeSystem: request.includeSystem,
|
|
9
|
-
includeSynthetic: request.includeSynthetic,
|
|
10
|
-
orderingSpecifier: request.orderingSpecifier
|
|
11
|
-
})
|
|
12
|
-
|
|
13
|
-
export const toExportBody = (request: Soda3ExportRequest) => ({
|
|
14
|
-
query: request.query,
|
|
15
|
-
parameters: request.parameters,
|
|
16
|
-
timeout: request.timeout,
|
|
17
|
-
serializationOptions: request.serializationOptions
|
|
18
|
-
})
|
|
19
|
-
|
|
20
|
-
export const rangeHeader = (request: Soda3ExportRequest) =>
|
|
21
|
-
request.range === undefined
|
|
22
|
-
? undefined
|
|
23
|
-
: `bytes=${request.range.start}-${request.range.end ?? ""}`
|
package/src/internal/http.ts
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import * as Effect from "effect/Effect"
|
|
2
|
-
import * as Stream from "effect/Stream"
|
|
3
|
-
import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
|
|
4
|
-
import type * as HttpClientError from "effect/unstable/http/HttpClientError"
|
|
5
|
-
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
6
|
-
import { ProviderHttpError } from "../errors"
|
|
7
|
-
|
|
8
|
-
export const Soda3HttpLive = FetchHttpClient.layer
|
|
9
|
-
|
|
10
|
-
export const retryableStatus = (status: number) => status === 429 || status >= 500
|
|
11
|
-
|
|
12
|
-
const httpStatusText = (status: number) => `HTTP ${status}`
|
|
13
|
-
|
|
14
|
-
export const toWebResponse = (response: HttpClientResponse.HttpClientResponse) =>
|
|
15
|
-
Effect.map(
|
|
16
|
-
Stream.toReadableStreamEffect(response.stream),
|
|
17
|
-
(body) =>
|
|
18
|
-
new Response(body, {
|
|
19
|
-
status: response.status,
|
|
20
|
-
headers: response.headers
|
|
21
|
-
})
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
export const mapHttpClientError = (operation: string, error: HttpClientError.HttpClientError) => {
|
|
25
|
-
const response = error.response
|
|
26
|
-
|
|
27
|
-
if (response !== undefined) {
|
|
28
|
-
return ProviderHttpError.make({
|
|
29
|
-
operation,
|
|
30
|
-
status: response.status,
|
|
31
|
-
statusText: httpStatusText(response.status),
|
|
32
|
-
retryable: retryableStatus(response.status)
|
|
33
|
-
})
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
return ProviderHttpError.make({
|
|
37
|
-
operation,
|
|
38
|
-
status: 0,
|
|
39
|
-
statusText: error.message,
|
|
40
|
-
retryable: true
|
|
41
|
-
})
|
|
42
|
-
}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
Soda3CatalogSearchResponse,
|
|
3
|
-
Soda3QueryResponse,
|
|
4
|
-
Soda3Row
|
|
5
|
-
} from "@nyc-transit-kit/contracts/soda3"
|
|
6
|
-
import * as Effect from "effect/Effect"
|
|
7
|
-
import * as Schema from "effect/Schema"
|
|
8
|
-
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
9
|
-
import { ProviderContractError } from "../errors"
|
|
10
|
-
|
|
11
|
-
const providerRows = Schema.Array(Soda3Row)
|
|
12
|
-
|
|
13
|
-
export const decodeJson = (operation: string, response: HttpClientResponse.HttpClientResponse) =>
|
|
14
|
-
response.json.pipe(
|
|
15
|
-
Effect.catchTag("HttpClientError", () =>
|
|
16
|
-
Effect.fail(
|
|
17
|
-
ProviderContractError.make({
|
|
18
|
-
operation,
|
|
19
|
-
message: "Provider response was not valid JSON"
|
|
20
|
-
})
|
|
21
|
-
)
|
|
22
|
-
)
|
|
23
|
-
)
|
|
24
|
-
|
|
25
|
-
export const decodeQueryResponse = (input: unknown) =>
|
|
26
|
-
Schema.decodeUnknownEffect(Soda3QueryResponse)(input).pipe(
|
|
27
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
28
|
-
Schema.decodeUnknownEffect(providerRows)(input).pipe(
|
|
29
|
-
Effect.map((rows) =>
|
|
30
|
-
Soda3QueryResponse.make({
|
|
31
|
-
rows
|
|
32
|
-
})
|
|
33
|
-
),
|
|
34
|
-
Effect.catchTag("SchemaError", () =>
|
|
35
|
-
Effect.fail(
|
|
36
|
-
ProviderContractError.make({
|
|
37
|
-
operation: "query",
|
|
38
|
-
message: error.message
|
|
39
|
-
})
|
|
40
|
-
)
|
|
41
|
-
)
|
|
42
|
-
)
|
|
43
|
-
)
|
|
44
|
-
)
|
|
45
|
-
|
|
46
|
-
export const decodeCatalogSearchResponse = (input: unknown) =>
|
|
47
|
-
Schema.decodeUnknownEffect(Soda3CatalogSearchResponse)(input).pipe(
|
|
48
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
49
|
-
Effect.fail(
|
|
50
|
-
ProviderContractError.make({
|
|
51
|
-
operation: "catalog",
|
|
52
|
-
message: error.message
|
|
53
|
-
})
|
|
54
|
-
)
|
|
55
|
-
)
|
|
56
|
-
)
|
package/src/internal/services.ts
DELETED
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
import * as Context from "effect/Context"
|
|
2
|
-
import * as Layer from "effect/Layer"
|
|
3
|
-
import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
|
|
4
|
-
import { Soda3HttpLive } from "./http"
|
|
5
|
-
|
|
6
|
-
export interface Soda3ClientConfigShape {
|
|
7
|
-
readonly appToken?: string
|
|
8
|
-
readonly retryTimes?: number
|
|
9
|
-
readonly retryDelayMs?: number
|
|
10
|
-
readonly timeoutMs?: number
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export type FetchImplementation = (
|
|
14
|
-
input: string | URL | globalThis.Request,
|
|
15
|
-
init?: globalThis.RequestInit
|
|
16
|
-
) => Promise<globalThis.Response>
|
|
17
|
-
|
|
18
|
-
export interface Soda3LayerOptions extends Soda3ClientConfigShape {
|
|
19
|
-
readonly fetch?: FetchImplementation
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export class Soda3ClientConfig extends Context.Service<Soda3ClientConfig, Soda3ClientConfigShape>()(
|
|
23
|
-
"@nyc-transit-kit/soda3/Soda3ClientConfig"
|
|
24
|
-
) {
|
|
25
|
-
static readonly Default = Layer.succeed(Soda3ClientConfig)({
|
|
26
|
-
retryTimes: 0
|
|
27
|
-
})
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export const soda3ConfigLayer = (options: Soda3ClientConfigShape = {}) =>
|
|
31
|
-
Layer.succeed(Soda3ClientConfig)({
|
|
32
|
-
retryTimes: options.retryTimes ?? 0,
|
|
33
|
-
...(options.appToken === undefined ? {} : { appToken: options.appToken }),
|
|
34
|
-
...(options.retryDelayMs === undefined ? {} : { retryDelayMs: options.retryDelayMs }),
|
|
35
|
-
...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs })
|
|
36
|
-
})
|
|
37
|
-
|
|
38
|
-
export const soda3FetchLayer = (fetchImplementation?: FetchImplementation) =>
|
|
39
|
-
fetchImplementation === undefined
|
|
40
|
-
? Soda3HttpLive
|
|
41
|
-
: FetchHttpClient.layer.pipe(
|
|
42
|
-
Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fetchImplementation as typeof fetch))
|
|
43
|
-
)
|
|
44
|
-
|
|
45
|
-
export const soda3Layer = (options: Soda3LayerOptions = {}) =>
|
|
46
|
-
Layer.mergeAll(soda3ConfigLayer(options), soda3FetchLayer(options.fetch))
|
|
47
|
-
|
|
48
|
-
export const Soda3Live = soda3Layer()
|