@nyc-transit-kit/soda3 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -21
- package/package.json +13 -18
- package/src/Dataset.ts +72 -0
- package/src/Discovery.ts +107 -0
- package/src/Socrata.ts +115 -0
- package/src/Soda3Client.ts +224 -0
- package/src/Soql.ts +200 -0
- package/src/index.ts +5 -73
- package/src/catalog.ts +0 -33
- package/src/client.ts +0 -44
- package/src/endpoints.ts +0 -77
- package/src/errors.ts +0 -58
- package/src/export.ts +0 -46
- package/src/internal/decode.ts +0 -12
- package/src/internal/transport.ts +0 -161
- package/src/metadata.ts +0 -37
- package/src/query.ts +0 -138
- package/src/soql.ts +0 -214
- package/src/transport.ts +0 -145
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
Soda3CatalogSearchResponse,
|
|
3
|
-
type Soda3ExportRequest,
|
|
4
|
-
type Soda3QueryRequest,
|
|
5
|
-
Soda3QueryResponse,
|
|
6
|
-
Soda3Row
|
|
7
|
-
} from "@nyc-transit-kit/contracts/soda3"
|
|
8
|
-
import * as Effect from "effect/Effect"
|
|
9
|
-
import * as Schema from "effect/Schema"
|
|
10
|
-
import * as HttpClient from "effect/unstable/http/HttpClient"
|
|
11
|
-
import type * as HttpClientError from "effect/unstable/http/HttpClientError"
|
|
12
|
-
import type * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
13
|
-
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
14
|
-
import { Soda3ClientConfig, type Soda3ClientConfigShape } from "../client"
|
|
15
|
-
import type { Soda3ClientError } from "../errors"
|
|
16
|
-
import {
|
|
17
|
-
isRetryableProviderError,
|
|
18
|
-
ProviderContractError,
|
|
19
|
-
ProviderHttpError,
|
|
20
|
-
RetryExhaustedError,
|
|
21
|
-
TimeoutError
|
|
22
|
-
} from "../errors"
|
|
23
|
-
import {
|
|
24
|
-
httpFailureInfo,
|
|
25
|
-
httpStatusText,
|
|
26
|
-
isOkStatus,
|
|
27
|
-
retryableStatus,
|
|
28
|
-
withRequestPolicy
|
|
29
|
-
} from "../transport"
|
|
30
|
-
|
|
31
|
-
export { retryableStatus, toWebResponse } from "../transport"
|
|
32
|
-
|
|
33
|
-
const providerRows = Schema.Array(Soda3Row)
|
|
34
|
-
|
|
35
|
-
const decodeUnknown = <S extends Schema.Top>(schema: S, input: unknown) =>
|
|
36
|
-
Schema.decodeUnknownEffect(schema)(input)
|
|
37
|
-
|
|
38
|
-
export const addHeaders = (config: Soda3ClientConfigShape) => ({
|
|
39
|
-
accept: "application/json",
|
|
40
|
-
...(config.appToken !== undefined && config.appToken.length > 0
|
|
41
|
-
? { "X-App-Token": config.appToken }
|
|
42
|
-
: {})
|
|
43
|
-
})
|
|
44
|
-
|
|
45
|
-
export const toQueryBody = (request: Soda3QueryRequest) => ({
|
|
46
|
-
query: request.query,
|
|
47
|
-
page: request.page,
|
|
48
|
-
parameters: request.parameters,
|
|
49
|
-
timeout: request.timeout,
|
|
50
|
-
includeSystem: request.includeSystem,
|
|
51
|
-
includeSynthetic: request.includeSynthetic,
|
|
52
|
-
orderingSpecifier: request.orderingSpecifier
|
|
53
|
-
})
|
|
54
|
-
|
|
55
|
-
export const toExportBody = (request: Soda3ExportRequest) => ({
|
|
56
|
-
query: request.query,
|
|
57
|
-
parameters: request.parameters,
|
|
58
|
-
timeout: request.timeout,
|
|
59
|
-
serializationOptions: request.serializationOptions
|
|
60
|
-
})
|
|
61
|
-
|
|
62
|
-
export const rangeHeader = (request: Soda3ExportRequest) =>
|
|
63
|
-
request.range === undefined
|
|
64
|
-
? undefined
|
|
65
|
-
: `bytes=${request.range.start}-${request.range.end ?? ""}`
|
|
66
|
-
|
|
67
|
-
export const mapHttpClientError = (operation: string, error: HttpClientError.HttpClientError) => {
|
|
68
|
-
const info = httpFailureInfo(error)
|
|
69
|
-
|
|
70
|
-
return ProviderHttpError.make({
|
|
71
|
-
operation,
|
|
72
|
-
status: info.status,
|
|
73
|
-
statusText: info.statusText,
|
|
74
|
-
retryable: info.status === 0 || retryableStatus(info.status)
|
|
75
|
-
})
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
export const decodeJson = (operation: string, response: HttpClientResponse.HttpClientResponse) =>
|
|
79
|
-
response.json.pipe(
|
|
80
|
-
Effect.catchTag("HttpClientError", () =>
|
|
81
|
-
Effect.fail(
|
|
82
|
-
ProviderContractError.make({
|
|
83
|
-
operation,
|
|
84
|
-
message: "Provider response was not valid JSON"
|
|
85
|
-
})
|
|
86
|
-
)
|
|
87
|
-
)
|
|
88
|
-
)
|
|
89
|
-
|
|
90
|
-
export const decodeQueryResponse = (input: unknown) =>
|
|
91
|
-
decodeUnknown(Soda3QueryResponse, input).pipe(
|
|
92
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
93
|
-
decodeUnknown(providerRows, input).pipe(
|
|
94
|
-
Effect.map((rows) =>
|
|
95
|
-
Soda3QueryResponse.make({
|
|
96
|
-
rows
|
|
97
|
-
})
|
|
98
|
-
),
|
|
99
|
-
Effect.catchTag("SchemaError", () =>
|
|
100
|
-
Effect.fail(
|
|
101
|
-
ProviderContractError.make({
|
|
102
|
-
operation: "query",
|
|
103
|
-
message: error.message
|
|
104
|
-
})
|
|
105
|
-
)
|
|
106
|
-
)
|
|
107
|
-
)
|
|
108
|
-
)
|
|
109
|
-
)
|
|
110
|
-
|
|
111
|
-
export const decodeCatalogSearchResponse = (input: unknown) =>
|
|
112
|
-
decodeUnknown(Soda3CatalogSearchResponse, input).pipe(
|
|
113
|
-
Effect.catchTag("SchemaError", (error) =>
|
|
114
|
-
Effect.fail(
|
|
115
|
-
ProviderContractError.make({
|
|
116
|
-
operation: "catalog",
|
|
117
|
-
message: error.message
|
|
118
|
-
})
|
|
119
|
-
)
|
|
120
|
-
)
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
export const requestWithPolicy = <A>(
|
|
124
|
-
operation: string,
|
|
125
|
-
effect: Effect.Effect<A, Soda3ClientError, HttpClient.HttpClient | Soda3ClientConfig>
|
|
126
|
-
) =>
|
|
127
|
-
Effect.gen(function* () {
|
|
128
|
-
const config = yield* Soda3ClientConfig
|
|
129
|
-
return yield* withRequestPolicy(config, effect, {
|
|
130
|
-
isRetryable: isRetryableProviderError,
|
|
131
|
-
onExhausted: (attempts, error) =>
|
|
132
|
-
RetryExhaustedError.make({
|
|
133
|
-
operation,
|
|
134
|
-
attempts,
|
|
135
|
-
message: `Retry attempts exhausted after provider status ${error.status}`
|
|
136
|
-
}),
|
|
137
|
-
onTimeout: (timeoutMs) =>
|
|
138
|
-
TimeoutError.make({
|
|
139
|
-
operation,
|
|
140
|
-
timeoutMs
|
|
141
|
-
})
|
|
142
|
-
})
|
|
143
|
-
})
|
|
144
|
-
|
|
145
|
-
export const executeRequest = (operation: string, request: HttpClientRequest.HttpClientRequest) =>
|
|
146
|
-
Effect.gen(function* () {
|
|
147
|
-
const response = yield* HttpClient.execute(request).pipe(
|
|
148
|
-
Effect.mapError((error) => mapHttpClientError(operation, error))
|
|
149
|
-
)
|
|
150
|
-
|
|
151
|
-
if (!isOkStatus(response.status)) {
|
|
152
|
-
return yield* ProviderHttpError.make({
|
|
153
|
-
operation,
|
|
154
|
-
status: response.status,
|
|
155
|
-
statusText: httpStatusText(response.status),
|
|
156
|
-
retryable: retryableStatus(response.status)
|
|
157
|
-
})
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
return response
|
|
161
|
-
})
|
package/src/metadata.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
type Soda3CatalogResultFragment,
|
|
3
|
-
Soda3DatasetMetadataRequest,
|
|
4
|
-
type Soda3DatasetMetadataRequestInput
|
|
5
|
-
} from "@nyc-transit-kit/contracts/soda3"
|
|
6
|
-
import * as Effect from "effect/Effect"
|
|
7
|
-
import { catalogSearch } from "./catalog"
|
|
8
|
-
import { ProviderContractError } from "./errors"
|
|
9
|
-
import { decodeInput } from "./internal/decode"
|
|
10
|
-
|
|
11
|
-
export type { Soda3DatasetMetadataRequestInput }
|
|
12
|
-
|
|
13
|
-
const decodeDatasetMetadataRequest = decodeInput("metadata", Soda3DatasetMetadataRequest)
|
|
14
|
-
|
|
15
|
-
const matchingResult = (results: ReadonlyArray<Soda3CatalogResultFragment>, datasetId: string) =>
|
|
16
|
-
results.find((result) => String(result.resource.id) === datasetId)
|
|
17
|
-
|
|
18
|
-
export const fetchDatasetMetadata = Effect.fn("Soda3.fetchDatasetMetadata")(function* (
|
|
19
|
-
input: Soda3DatasetMetadataRequestInput
|
|
20
|
-
) {
|
|
21
|
-
const request = yield* decodeDatasetMetadataRequest(input)
|
|
22
|
-
const response = yield* catalogSearch({
|
|
23
|
-
domain: request.domain,
|
|
24
|
-
ids: [request.datasetId],
|
|
25
|
-
limit: 1
|
|
26
|
-
})
|
|
27
|
-
const result = matchingResult(response.results, String(request.datasetId))
|
|
28
|
-
|
|
29
|
-
if (result === undefined) {
|
|
30
|
-
return yield* ProviderContractError.make({
|
|
31
|
-
operation: "metadata",
|
|
32
|
-
message: `Discovery metadata response did not include dataset ${request.datasetId}`
|
|
33
|
-
})
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
return result
|
|
37
|
-
})
|
package/src/query.ts
DELETED
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
PositiveInteger,
|
|
3
|
-
type Soda3QueryRequestInput,
|
|
4
|
-
type Soda3Row
|
|
5
|
-
} from "@nyc-transit-kit/contracts/soda3"
|
|
6
|
-
import * as Effect from "effect/Effect"
|
|
7
|
-
import * as HttpClientRequest from "effect/unstable/http/HttpClientRequest"
|
|
8
|
-
import { Soda3ClientConfig } from "./client"
|
|
9
|
-
import { buildQueryUrl, decodeSoda3QueryRequest } from "./endpoints"
|
|
10
|
-
import { InvalidInputError, ProviderContractError } from "./errors"
|
|
11
|
-
import { decodeInput } from "./internal/decode"
|
|
12
|
-
import {
|
|
13
|
-
addHeaders,
|
|
14
|
-
decodeJson,
|
|
15
|
-
decodeQueryResponse,
|
|
16
|
-
executeRequest,
|
|
17
|
-
requestWithPolicy,
|
|
18
|
-
toQueryBody
|
|
19
|
-
} from "./internal/transport"
|
|
20
|
-
|
|
21
|
-
export type { Soda3QueryRequestInput }
|
|
22
|
-
|
|
23
|
-
export type Soda3QueryAllRowsInput = Omit<Soda3QueryRequestInput, "page"> & {
|
|
24
|
-
readonly pageSize?: number
|
|
25
|
-
readonly maxRows?: number
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
export type Soda3CountRowsInput = {
|
|
29
|
-
readonly domain: string
|
|
30
|
-
readonly datasetId: string
|
|
31
|
-
readonly where?: string
|
|
32
|
-
readonly parameters?: Readonly<Record<string, unknown>>
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const decodeHelperPositiveInteger = (name: string, value: number) =>
|
|
36
|
-
decodeInput(
|
|
37
|
-
"query",
|
|
38
|
-
PositiveInteger
|
|
39
|
-
)(value).pipe(
|
|
40
|
-
Effect.mapError((error) =>
|
|
41
|
-
InvalidInputError.make({
|
|
42
|
-
operation: "query",
|
|
43
|
-
message: `Invalid ${name}: ${error.message}`
|
|
44
|
-
})
|
|
45
|
-
)
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
const decodeOptionalHelperPositiveInteger = (name: string, value: number | undefined) =>
|
|
49
|
-
value === undefined ? Effect.void : decodeHelperPositiveInteger(name, value)
|
|
50
|
-
|
|
51
|
-
const decodeCountValue = (value: unknown) => {
|
|
52
|
-
const parsed =
|
|
53
|
-
typeof value === "number" ? value : typeof value === "string" ? Number(value) : Number.NaN
|
|
54
|
-
|
|
55
|
-
return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
const countColumnName = "count"
|
|
59
|
-
|
|
60
|
-
export const queryRows = Effect.fn("Soda3.queryRows")(function* (input: Soda3QueryRequestInput) {
|
|
61
|
-
const config = yield* Soda3ClientConfig
|
|
62
|
-
const queryRequest = yield* decodeSoda3QueryRequest(input)
|
|
63
|
-
const url = yield* buildQueryUrl(queryRequest)
|
|
64
|
-
const request = yield* HttpClientRequest.post(url, {
|
|
65
|
-
headers: addHeaders(config)
|
|
66
|
-
}).pipe(
|
|
67
|
-
HttpClientRequest.bodyJson(toQueryBody(queryRequest)),
|
|
68
|
-
Effect.mapError(() =>
|
|
69
|
-
InvalidInputError.make({
|
|
70
|
-
operation: "query",
|
|
71
|
-
message: "SODA3 query body was not JSON serializable"
|
|
72
|
-
})
|
|
73
|
-
)
|
|
74
|
-
)
|
|
75
|
-
|
|
76
|
-
return yield* requestWithPolicy(
|
|
77
|
-
"query",
|
|
78
|
-
executeRequest("query", request).pipe(
|
|
79
|
-
Effect.flatMap((response) => decodeJson("query", response)),
|
|
80
|
-
Effect.flatMap(decodeQueryResponse)
|
|
81
|
-
)
|
|
82
|
-
)
|
|
83
|
-
})
|
|
84
|
-
|
|
85
|
-
export const queryAllRows = Effect.fn("Soda3.queryAllRows")(function* (
|
|
86
|
-
input: Soda3QueryAllRowsInput
|
|
87
|
-
) {
|
|
88
|
-
const { pageSize: rawPageSize = 1000, maxRows: rawMaxRows, ...queryInput } = input
|
|
89
|
-
const pageSize = yield* decodeHelperPositiveInteger("pageSize", rawPageSize)
|
|
90
|
-
const maxRows = yield* decodeOptionalHelperPositiveInteger("maxRows", rawMaxRows)
|
|
91
|
-
const rows: Array<Soda3Row> = []
|
|
92
|
-
let pageNumber = 1
|
|
93
|
-
|
|
94
|
-
while (true) {
|
|
95
|
-
const response = yield* queryRows({
|
|
96
|
-
...queryInput,
|
|
97
|
-
page: {
|
|
98
|
-
pageNumber,
|
|
99
|
-
pageSize
|
|
100
|
-
}
|
|
101
|
-
})
|
|
102
|
-
rows.push(...response.rows)
|
|
103
|
-
|
|
104
|
-
if (maxRows !== undefined && rows.length > maxRows) {
|
|
105
|
-
return yield* InvalidInputError.make({
|
|
106
|
-
operation: "query",
|
|
107
|
-
message: `SODA3 query exceeded maxRows=${maxRows}; narrow the query or raise maxRows`
|
|
108
|
-
})
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
if (response.rows.length < pageSize) {
|
|
112
|
-
return rows
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
pageNumber += 1
|
|
116
|
-
}
|
|
117
|
-
})
|
|
118
|
-
|
|
119
|
-
export const countRows = Effect.fn("Soda3.countRows")(function* (input: Soda3CountRowsInput) {
|
|
120
|
-
const query = `SELECT count(*)${input.where === undefined ? "" : ` WHERE ${input.where}`}`
|
|
121
|
-
const response = yield* queryRows({
|
|
122
|
-
domain: input.domain,
|
|
123
|
-
datasetId: input.datasetId,
|
|
124
|
-
query,
|
|
125
|
-
...(input.parameters === undefined ? {} : { parameters: input.parameters }),
|
|
126
|
-
includeSynthetic: false
|
|
127
|
-
})
|
|
128
|
-
const count = decodeCountValue(response.rows[0]?.[countColumnName])
|
|
129
|
-
|
|
130
|
-
if (count === undefined) {
|
|
131
|
-
return yield* ProviderContractError.make({
|
|
132
|
-
operation: "query",
|
|
133
|
-
message: "SODA3 count(*) response did not include a numeric count"
|
|
134
|
-
})
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return count
|
|
138
|
-
})
|
package/src/soql.ts
DELETED
|
@@ -1,214 +0,0 @@
|
|
|
1
|
-
import { IsoDate } from "@nyc-transit-kit/contracts/ids"
|
|
2
|
-
import { PositiveInteger } from "@nyc-transit-kit/contracts/soda3"
|
|
3
|
-
import * as Effect from "effect/Effect"
|
|
4
|
-
import * as Schema from "effect/Schema"
|
|
5
|
-
import { InvalidInputError } from "./errors"
|
|
6
|
-
import { decodeInput } from "./internal/decode"
|
|
7
|
-
export const soqlSelectAll = "SELECT *"
|
|
8
|
-
export type SoqlParameters = Readonly<Record<string, unknown>>
|
|
9
|
-
export type SoqlFragment = Readonly<{ text: string; parameters?: SoqlParameters }>
|
|
10
|
-
export type SocrataTimestampWindow = Readonly<{ start: string; end: string }>
|
|
11
|
-
export type SocrataYearMonth = Readonly<{ year: number; month: number }>
|
|
12
|
-
const fn = Effect.fn
|
|
13
|
-
const identifierPattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
14
|
-
const parameterPattern = /^[A-Za-z_][A-Za-z0-9_]*$/
|
|
15
|
-
const datePattern = /^(\d{4})-(\d{2})-(\d{2})$/
|
|
16
|
-
const patternString = (pattern: RegExp, message: string) =>
|
|
17
|
-
Schema.String.pipe(Schema.check(Schema.isPattern(pattern, { message })))
|
|
18
|
-
const SoqlIdentifier = patternString(
|
|
19
|
-
identifierPattern,
|
|
20
|
-
"Expected a SoQL identifier such as route_id"
|
|
21
|
-
)
|
|
22
|
-
const SoqlParameterName = patternString(
|
|
23
|
-
parameterPattern,
|
|
24
|
-
"Expected a SoQL parameter name such as route"
|
|
25
|
-
)
|
|
26
|
-
const SoqlSortDirection = Schema.Literals(["ASC", "DESC"])
|
|
27
|
-
export type SoqlSortDirection = typeof SoqlSortDirection.Type
|
|
28
|
-
const intRange = (minimum: number, maximum: number) =>
|
|
29
|
-
Schema.Int.pipe(
|
|
30
|
-
Schema.check(Schema.isGreaterThanOrEqualTo(minimum)),
|
|
31
|
-
Schema.check(Schema.isLessThanOrEqualTo(maximum))
|
|
32
|
-
)
|
|
33
|
-
const SocrataYear = intRange(1900, 9999)
|
|
34
|
-
const SocrataMonth = intRange(1, 12)
|
|
35
|
-
const SocrataYearMonthSchema = Schema.Struct({ year: SocrataYear, month: SocrataMonth })
|
|
36
|
-
const invalidQueryInput = (message: string) =>
|
|
37
|
-
InvalidInputError.make({ operation: "query", message })
|
|
38
|
-
const decode = <S extends Schema.Top>(schema: S) => decodeInput("query", schema)
|
|
39
|
-
const decodeLimit = decode(PositiveInteger)
|
|
40
|
-
const decodeIdentifier = decode(SoqlIdentifier)
|
|
41
|
-
const decodeParameterName = decode(SoqlParameterName)
|
|
42
|
-
const decodeSortDirection = decode(SoqlSortDirection)
|
|
43
|
-
const decodeIsoDate = decode(IsoDate)
|
|
44
|
-
const decodeYear = decode(SocrataYear)
|
|
45
|
-
const decodeMonth = decode(SocrataMonth)
|
|
46
|
-
const decodeYearMonth = decode(SocrataYearMonthSchema)
|
|
47
|
-
const fragment = (text: string, parameters?: SoqlParameters): SoqlFragment =>
|
|
48
|
-
parameters === undefined ? { text } : { text, parameters }
|
|
49
|
-
const mergeParameters = (fragments: ReadonlyArray<SoqlFragment>) => {
|
|
50
|
-
const parameters: Record<string, unknown> = {}
|
|
51
|
-
for (const part of fragments) {
|
|
52
|
-
for (const [key, value] of Object.entries(part.parameters ?? {})) {
|
|
53
|
-
if (Object.hasOwn(parameters, key)) {
|
|
54
|
-
return Effect.fail(invalidQueryInput(`Duplicate SoQL parameter name: ${key}`))
|
|
55
|
-
}
|
|
56
|
-
parameters[key] = value
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
return Effect.succeed(Object.keys(parameters).length === 0 ? undefined : parameters)
|
|
60
|
-
}
|
|
61
|
-
const dateMessage = "Expected an ISO calendar date in YYYY-MM-DD format"
|
|
62
|
-
const pad = (value: number, length: number) => value.toString().padStart(length, "0")
|
|
63
|
-
const timestampFromParts = (year: number, month: number, day: number) =>
|
|
64
|
-
`${pad(year, 4)}-${pad(month, 2)}-${pad(day, 2)}T00:00:00`
|
|
65
|
-
const datePartsOf = (value: string) => {
|
|
66
|
-
const [, yearText, monthText, dayText] = datePattern.exec(value) ?? []
|
|
67
|
-
return yearText === undefined || monthText === undefined || dayText === undefined
|
|
68
|
-
? Effect.fail(invalidQueryInput(dateMessage))
|
|
69
|
-
: Effect.succeed({ year: +yearText, month: +monthText, day: +dayText })
|
|
70
|
-
}
|
|
71
|
-
export const soqlLimit = fn("Soda3.soqlLimit")(function* (query: string, limit: unknown) {
|
|
72
|
-
return `${query} LIMIT ${yield* decodeLimit(limit)}`
|
|
73
|
-
})
|
|
74
|
-
export const soqlIdentifier = fn("Soda3.soqlIdentifier")(function* (input: unknown) {
|
|
75
|
-
return yield* decodeIdentifier(input)
|
|
76
|
-
})
|
|
77
|
-
export const soqlParameterName = fn("Soda3.soqlParameterName")(function* (input: unknown) {
|
|
78
|
-
return yield* decodeParameterName(input)
|
|
79
|
-
})
|
|
80
|
-
export const soqlParameter = fn("Soda3.soqlParameter")(function* (name: unknown, value: unknown) {
|
|
81
|
-
const key = yield* decodeParameterName(name)
|
|
82
|
-
return fragment(`:${key}`, { [key]: value })
|
|
83
|
-
})
|
|
84
|
-
export const soqlEq = fn("Soda3.soqlEq")(function* (
|
|
85
|
-
column: unknown,
|
|
86
|
-
parameterName: unknown,
|
|
87
|
-
value: unknown
|
|
88
|
-
) {
|
|
89
|
-
const col = yield* decodeIdentifier(column)
|
|
90
|
-
const key = yield* decodeParameterName(parameterName)
|
|
91
|
-
return fragment(`${col} = :${key}`, { [key]: value })
|
|
92
|
-
})
|
|
93
|
-
export const soqlIn = fn("Soda3.soqlIn")(function* (
|
|
94
|
-
column: unknown,
|
|
95
|
-
parameterBaseName: unknown,
|
|
96
|
-
values: ReadonlyArray<unknown>
|
|
97
|
-
) {
|
|
98
|
-
if (values.length === 0) {
|
|
99
|
-
return yield* invalidQueryInput("SoQL IN predicates require at least one value")
|
|
100
|
-
}
|
|
101
|
-
const col = yield* decodeIdentifier(column)
|
|
102
|
-
const base = yield* decodeParameterName(parameterBaseName)
|
|
103
|
-
const parameters: Record<string, unknown> = {}
|
|
104
|
-
const placeholders: Array<string> = []
|
|
105
|
-
for (let index = 0; index < values.length; index += 1) {
|
|
106
|
-
const key = `${base}_${index + 1}`
|
|
107
|
-
parameters[key] = values[index]
|
|
108
|
-
placeholders.push(`:${key}`)
|
|
109
|
-
}
|
|
110
|
-
return fragment(`${col} IN (${placeholders.join(", ")})`, parameters)
|
|
111
|
-
})
|
|
112
|
-
export const soqlIsNotNull = fn("Soda3.soqlIsNotNull")(function* (column: unknown) {
|
|
113
|
-
return fragment(`${yield* decodeIdentifier(column)} IS NOT NULL`)
|
|
114
|
-
})
|
|
115
|
-
export const soqlAnd = fn("Soda3.soqlAnd")(function* (fragments: ReadonlyArray<SoqlFragment>) {
|
|
116
|
-
if (fragments.length === 0) {
|
|
117
|
-
return yield* invalidQueryInput("SoQL AND composition requires at least one fragment")
|
|
118
|
-
}
|
|
119
|
-
return fragment(
|
|
120
|
-
fragments.map((part) => `(${part.text})`).join(" AND "),
|
|
121
|
-
yield* mergeParameters(fragments)
|
|
122
|
-
)
|
|
123
|
-
})
|
|
124
|
-
export const soqlOrderBy = fn("Soda3.soqlOrderBy")(function* (
|
|
125
|
-
column: unknown,
|
|
126
|
-
direction: unknown = "ASC"
|
|
127
|
-
) {
|
|
128
|
-
const col = yield* decodeIdentifier(column)
|
|
129
|
-
return fragment(`ORDER BY ${col} ${yield* decodeSortDirection(direction)}`)
|
|
130
|
-
})
|
|
131
|
-
export const socrataTimestamp = fn("Soda3.socrataTimestamp")(function* (date: string | Date) {
|
|
132
|
-
if (date instanceof Date) {
|
|
133
|
-
if (Number.isNaN(date.getTime())) {
|
|
134
|
-
return yield* invalidQueryInput("Expected a valid Date")
|
|
135
|
-
}
|
|
136
|
-
return timestampFromParts(date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate())
|
|
137
|
-
}
|
|
138
|
-
const decodedDate = yield* decodeIsoDate(date)
|
|
139
|
-
const parts = yield* datePartsOf(decodedDate)
|
|
140
|
-
return timestampFromParts(parts.year, parts.month, parts.day)
|
|
141
|
-
})
|
|
142
|
-
export const socrataMonthWindow = fn("Soda3.socrataMonthWindow")(function* (
|
|
143
|
-
year: unknown,
|
|
144
|
-
month: unknown
|
|
145
|
-
) {
|
|
146
|
-
const y = yield* decodeYear(year)
|
|
147
|
-
const m = yield* decodeMonth(month)
|
|
148
|
-
const endYear = m === 12 ? y + 1 : y
|
|
149
|
-
const endMonth = m === 12 ? 1 : m + 1
|
|
150
|
-
return {
|
|
151
|
-
start: timestampFromParts(y, m, 1),
|
|
152
|
-
end: timestampFromParts(endYear, endMonth, 1)
|
|
153
|
-
}
|
|
154
|
-
})
|
|
155
|
-
export const socrataDateWindow = fn("Soda3.socrataDateWindow")(function* (
|
|
156
|
-
startDate: unknown,
|
|
157
|
-
endDate: unknown
|
|
158
|
-
) {
|
|
159
|
-
const start = yield* datePartsOf(yield* decodeIsoDate(startDate))
|
|
160
|
-
const end = yield* datePartsOf(yield* decodeIsoDate(endDate))
|
|
161
|
-
if (
|
|
162
|
-
Date.UTC(start.year, start.month - 1, start.day) >= Date.UTC(end.year, end.month - 1, end.day)
|
|
163
|
-
) {
|
|
164
|
-
return yield* invalidQueryInput("Socrata date windows require startDate before endDate")
|
|
165
|
-
}
|
|
166
|
-
return {
|
|
167
|
-
start: timestampFromParts(start.year, start.month, start.day),
|
|
168
|
-
end: timestampFromParts(end.year, end.month, end.day)
|
|
169
|
-
}
|
|
170
|
-
})
|
|
171
|
-
export const soqlTimestampRange = fn("Soda3.soqlTimestampRange")(function* (
|
|
172
|
-
column: unknown,
|
|
173
|
-
startParameterName: unknown,
|
|
174
|
-
endParameterName: unknown,
|
|
175
|
-
w: SocrataTimestampWindow
|
|
176
|
-
) {
|
|
177
|
-
const col = yield* decodeIdentifier(column)
|
|
178
|
-
const s = yield* decodeParameterName(startParameterName)
|
|
179
|
-
const e = yield* decodeParameterName(endParameterName)
|
|
180
|
-
if (s === e) {
|
|
181
|
-
return yield* invalidQueryInput(`Duplicate SoQL parameter name: ${s}`)
|
|
182
|
-
}
|
|
183
|
-
return fragment(`${col} >= :${s} AND ${col} < :${e}`, { [s]: w.start, [e]: w.end })
|
|
184
|
-
})
|
|
185
|
-
export const soqlMonthWindow = fn("Soda3.soqlMonthWindow")(function* (
|
|
186
|
-
column: unknown,
|
|
187
|
-
year: unknown,
|
|
188
|
-
month: unknown,
|
|
189
|
-
parameterBaseName: unknown
|
|
190
|
-
) {
|
|
191
|
-
const base = yield* decodeParameterName(parameterBaseName)
|
|
192
|
-
const window = yield* socrataMonthWindow(year, month)
|
|
193
|
-
return yield* soqlTimestampRange(column, `${base}_start`, `${base}_end`, window)
|
|
194
|
-
})
|
|
195
|
-
export const soqlYearMonthRange = fn("Soda3.soqlYearMonthRange")(function* (
|
|
196
|
-
yearColumn: unknown,
|
|
197
|
-
monthColumn: unknown,
|
|
198
|
-
start: unknown,
|
|
199
|
-
end: unknown
|
|
200
|
-
) {
|
|
201
|
-
const yearCol = yield* decodeIdentifier(yearColumn)
|
|
202
|
-
const monthCol = yield* decodeIdentifier(monthColumn)
|
|
203
|
-
const rs = yield* decodeYearMonth(start)
|
|
204
|
-
const re = yield* decodeYearMonth(end)
|
|
205
|
-
const startOrder = rs.year * 12 + rs.month
|
|
206
|
-
const endOrder = re.year * 12 + re.month
|
|
207
|
-
if (startOrder > endOrder) {
|
|
208
|
-
return yield* invalidQueryInput("SoQL year/month ranges require start before end")
|
|
209
|
-
}
|
|
210
|
-
return fragment(
|
|
211
|
-
`(${yearCol} > :start_year OR (${yearCol} = :start_year AND ${monthCol} >= :start_month)) AND (${yearCol} < :end_year OR (${yearCol} = :end_year AND ${monthCol} <= :end_month))`,
|
|
212
|
-
{ start_year: rs.year, start_month: rs.month, end_year: re.year, end_month: re.month }
|
|
213
|
-
)
|
|
214
|
-
})
|
package/src/transport.ts
DELETED
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
import * as Duration from "effect/Duration"
|
|
2
|
-
import * as Effect from "effect/Effect"
|
|
3
|
-
import * as Layer from "effect/Layer"
|
|
4
|
-
import * as Schedule from "effect/Schedule"
|
|
5
|
-
import * as Stream from "effect/Stream"
|
|
6
|
-
import * as FetchHttpClient from "effect/unstable/http/FetchHttpClient"
|
|
7
|
-
import type * as Headers from "effect/unstable/http/Headers"
|
|
8
|
-
import type * as HttpClientError from "effect/unstable/http/HttpClientError"
|
|
9
|
-
import type * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"
|
|
10
|
-
|
|
11
|
-
export type FetchImplementation = (
|
|
12
|
-
input: string | URL | globalThis.Request,
|
|
13
|
-
init?: globalThis.RequestInit
|
|
14
|
-
) => Promise<globalThis.Response>
|
|
15
|
-
|
|
16
|
-
export interface RequestPolicyConfig {
|
|
17
|
-
readonly retryTimes?: number
|
|
18
|
-
readonly retryDelayMs?: number
|
|
19
|
-
readonly timeoutMs?: number
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export const httpStatusText = (status: number) => `HTTP ${status}`
|
|
23
|
-
export const retryableStatus = (status: number) => status === 429 || status >= 500
|
|
24
|
-
export const isOkStatus = (status: number) => status >= 200 && status < 300
|
|
25
|
-
|
|
26
|
-
const retryPolicy = <E>(
|
|
27
|
-
retryTimes: number,
|
|
28
|
-
retryDelayMs: number | undefined,
|
|
29
|
-
isRetryable: (error: E) => boolean
|
|
30
|
-
) =>
|
|
31
|
-
retryDelayMs === undefined || retryDelayMs <= 0
|
|
32
|
-
? {
|
|
33
|
-
times: retryTimes,
|
|
34
|
-
while: isRetryable
|
|
35
|
-
}
|
|
36
|
-
: {
|
|
37
|
-
schedule: Schedule.exponential(Duration.millis(retryDelayMs)).pipe(
|
|
38
|
-
Schedule.both(Schedule.recurs(retryTimes))
|
|
39
|
-
),
|
|
40
|
-
while: isRetryable
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
export const withRequestPolicy = <A, E, R, Retryable extends E>(
|
|
44
|
-
config: RequestPolicyConfig,
|
|
45
|
-
effect: Effect.Effect<A, E, R>,
|
|
46
|
-
adapters: {
|
|
47
|
-
readonly isRetryable: (error: E) => error is Retryable
|
|
48
|
-
readonly onExhausted: (attempts: number, error: Retryable) => E
|
|
49
|
-
readonly onTimeout: (timeoutMs: number) => E
|
|
50
|
-
readonly retryableResult?: (value: A) => E | undefined
|
|
51
|
-
}
|
|
52
|
-
): Effect.Effect<A, E, R> => {
|
|
53
|
-
const retryTimes = config.retryTimes ?? 0
|
|
54
|
-
const effectWithRetryableResults =
|
|
55
|
-
retryTimes > 0 && adapters.retryableResult !== undefined
|
|
56
|
-
? effect.pipe(
|
|
57
|
-
Effect.flatMap((value) => {
|
|
58
|
-
const retryableError = adapters.retryableResult?.(value)
|
|
59
|
-
return retryableError === undefined
|
|
60
|
-
? Effect.succeed(value)
|
|
61
|
-
: Effect.fail(retryableError)
|
|
62
|
-
})
|
|
63
|
-
)
|
|
64
|
-
: effect
|
|
65
|
-
const retried =
|
|
66
|
-
retryTimes > 0
|
|
67
|
-
? effectWithRetryableResults.pipe(
|
|
68
|
-
Effect.retry(retryPolicy(retryTimes, config.retryDelayMs, adapters.isRetryable)),
|
|
69
|
-
Effect.catchIf(adapters.isRetryable, (error) =>
|
|
70
|
-
Effect.fail(adapters.onExhausted(retryTimes + 1, error))
|
|
71
|
-
)
|
|
72
|
-
)
|
|
73
|
-
: effectWithRetryableResults
|
|
74
|
-
const timeoutMs = config.timeoutMs
|
|
75
|
-
|
|
76
|
-
return timeoutMs === undefined
|
|
77
|
-
? retried
|
|
78
|
-
: retried.pipe(
|
|
79
|
-
Effect.timeoutOrElse({
|
|
80
|
-
duration: timeoutMs,
|
|
81
|
-
orElse: () => Effect.fail(adapters.onTimeout(timeoutMs))
|
|
82
|
-
})
|
|
83
|
-
)
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
export const httpFailureInfo = (error: HttpClientError.HttpClientError) => {
|
|
87
|
-
const response = error.response
|
|
88
|
-
|
|
89
|
-
return response === undefined
|
|
90
|
-
? {
|
|
91
|
-
status: 0,
|
|
92
|
-
statusText: error.message
|
|
93
|
-
}
|
|
94
|
-
: {
|
|
95
|
-
status: response.status,
|
|
96
|
-
statusText: httpStatusText(response.status)
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export const toWebResponse = (response: HttpClientResponse.HttpClientResponse) =>
|
|
101
|
-
Effect.map(
|
|
102
|
-
Stream.toReadableStreamEffect(response.stream),
|
|
103
|
-
(body) =>
|
|
104
|
-
new Response(body, {
|
|
105
|
-
status: response.status,
|
|
106
|
-
headers: response.headers
|
|
107
|
-
})
|
|
108
|
-
)
|
|
109
|
-
|
|
110
|
-
export const contentLengthOf = (headers: Headers.Headers) => {
|
|
111
|
-
const value = headers["content-length"]
|
|
112
|
-
if (value === undefined) {
|
|
113
|
-
return undefined
|
|
114
|
-
}
|
|
115
|
-
const parsed = Number(value)
|
|
116
|
-
return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
const shouldRedactSearchParameter = (name: string) => {
|
|
120
|
-
const lower = name.toLowerCase()
|
|
121
|
-
return lower.includes("key") || lower.includes("token") || lower.includes("secret")
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
export const redactedUrl = (url: string) => {
|
|
125
|
-
try {
|
|
126
|
-
const parsed = new URL(url)
|
|
127
|
-
|
|
128
|
-
for (const name of parsed.searchParams.keys()) {
|
|
129
|
-
if (shouldRedactSearchParameter(name)) {
|
|
130
|
-
parsed.searchParams.set(name, "REDACTED")
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
return parsed.toString()
|
|
135
|
-
} catch {
|
|
136
|
-
return "[invalid-url]"
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
export const fetchHttpClientLayer = (fetchImplementation?: FetchImplementation) =>
|
|
141
|
-
fetchImplementation === undefined
|
|
142
|
-
? FetchHttpClient.layer
|
|
143
|
-
: FetchHttpClient.layer.pipe(
|
|
144
|
-
Layer.provide(Layer.succeed(FetchHttpClient.Fetch, fetchImplementation as typeof fetch))
|
|
145
|
-
)
|