@synthient/synthient 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +198 -11
- package/README.md +407 -84
- package/dist/client.d.ts +98 -39
- package/dist/client.js +141 -90
- package/dist/errors.d.ts +72 -0
- package/dist/errors.js +122 -0
- package/dist/events.d.ts +217 -0
- package/dist/events.js +1 -0
- package/dist/feed.d.ts +75 -0
- package/dist/feed.js +192 -0
- package/dist/grpcschema.d.ts +56 -0
- package/dist/grpcschema.js +278 -0
- package/dist/index.d.ts +23 -5
- package/dist/index.js +20 -21
- package/dist/options.d.ts +5 -0
- package/dist/options.js +23 -0
- package/dist/transport.d.ts +76 -0
- package/dist/transport.js +328 -0
- package/dist/types.d.ts +248 -0
- package/dist/types.js +4 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +2 -0
- package/package.json +70 -12
- package/dist/constants.d.ts +0 -2
- package/dist/constants.js +0 -5
- package/dist/exceptions.d.ts +0 -33
- package/dist/exceptions.js +0 -48
- package/dist/models.d.ts +0 -191
- package/dist/models.js +0 -221
- package/dist/utils.d.ts +0 -15
- package/dist/utils.js +0 -29
- package/src/client.ts +0 -106
- package/src/constants.ts +0 -2
- package/src/exceptions.ts +0 -44
- package/src/index.ts +0 -5
- package/src/models.ts +0 -276
- package/src/utils.ts +0 -28
- package/tsconfig.json +0 -15
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import type { UnixTime } from "./types.ts";
|
|
2
|
+
/** New values may be added. */
|
|
3
|
+
export type ServiceCategory = "FREE_VPN" | "COMMERCIAL_VPN" | "ENTERPRISE_VPN" | "MOBILE_PROXY" | "BLOCKCHAIN_PROXY" | "RESIDENTIAL_PROXY" | "PUBLIC_PROXY" | "DATACENTER_PROXY" | "TOR_NODE" | "PRIVATE_RELAY" | "BOTNET" | "SEARCH_ENGINE" | "AI_CRAWLER" | "SOCIAL_MEDIA" | "UPTIME_MONITOR" | "LINK_PREVIEW" | "SEO_CRAWLER" | "WEB_ARCHIVER" | "WEBHOOK_PROVIDER" | "PAYMENT_PROCESSOR" | "INTERNET_SCANNER" | (string & {});
|
|
4
|
+
export interface ProxyEvent {
|
|
5
|
+
/** The proxy exit address. */
|
|
6
|
+
ip?: string;
|
|
7
|
+
/** The proxy service tag. */
|
|
8
|
+
provider?: string;
|
|
9
|
+
type?: ServiceCategory;
|
|
10
|
+
/** The ISO 3166-1 alpha-2 country of the address. */
|
|
11
|
+
country_code?: string;
|
|
12
|
+
asn?: number;
|
|
13
|
+
timestamp?: UnixTime;
|
|
14
|
+
}
|
|
15
|
+
export interface AnonymizerEvent {
|
|
16
|
+
/** The first address of the range. */
|
|
17
|
+
range_start?: string;
|
|
18
|
+
/** The last address of the range. */
|
|
19
|
+
range_end?: string;
|
|
20
|
+
/** The service tag. */
|
|
21
|
+
provider?: string;
|
|
22
|
+
type?: ServiceCategory;
|
|
23
|
+
/** The ISO 3166-1 alpha-2 country of `range_start`. */
|
|
24
|
+
country_code?: string;
|
|
25
|
+
/** The autonomous system number of `range_start`. */
|
|
26
|
+
asn?: number;
|
|
27
|
+
/** When the range was seen, in Unix milliseconds. */
|
|
28
|
+
timestamp?: UnixTime;
|
|
29
|
+
}
|
|
30
|
+
/** A torrent observed on the BitTorrent DHT or a tracker. */
|
|
31
|
+
export interface TorrentEvent {
|
|
32
|
+
/** The hex-encoded SHA-1 info hash. */
|
|
33
|
+
info_hash?: string;
|
|
34
|
+
name?: string;
|
|
35
|
+
magnet_uri?: string;
|
|
36
|
+
/** The total size of the torrent's files in bytes. */
|
|
37
|
+
total_size?: number;
|
|
38
|
+
/** The piece size in bytes. */
|
|
39
|
+
piece_length?: number;
|
|
40
|
+
file_count?: number;
|
|
41
|
+
files?: TorrentFile[];
|
|
42
|
+
peers?: TorrentPeer[];
|
|
43
|
+
timestamp?: UnixTime;
|
|
44
|
+
}
|
|
45
|
+
export interface TorrentFile {
|
|
46
|
+
path?: string;
|
|
47
|
+
/** The file size in bytes. */
|
|
48
|
+
length?: number;
|
|
49
|
+
}
|
|
50
|
+
export interface TorrentPeer {
|
|
51
|
+
ip?: string;
|
|
52
|
+
port?: number;
|
|
53
|
+
/** Where the peer was found, such as "DHT", "PEX", or "tracker". */
|
|
54
|
+
source?: string;
|
|
55
|
+
/** Whether the peer supports encrypted connections. */
|
|
56
|
+
encrypted?: boolean;
|
|
57
|
+
}
|
|
58
|
+
export interface JA4TEvent {
|
|
59
|
+
ip?: string;
|
|
60
|
+
ja4t?: string;
|
|
61
|
+
/** The service tag of the address. */
|
|
62
|
+
provider?: string;
|
|
63
|
+
type?: ServiceCategory;
|
|
64
|
+
/** The ISO 3166-1 alpha-2 country of the address. */
|
|
65
|
+
country_code?: string;
|
|
66
|
+
asn?: number;
|
|
67
|
+
timestamp?: UnixTime;
|
|
68
|
+
}
|
|
69
|
+
/** The Helios honeypot sensor that captured an event. */
|
|
70
|
+
export interface HeliosMeta {
|
|
71
|
+
/** The proxy exit address the traffic arrived through. */
|
|
72
|
+
proxy_ip?: string;
|
|
73
|
+
/** The sensor that captured the traffic. */
|
|
74
|
+
server?: string;
|
|
75
|
+
/** The sensor pool. */
|
|
76
|
+
pool_id?: string;
|
|
77
|
+
/** The proxy service the traffic arrived through. */
|
|
78
|
+
provider?: string;
|
|
79
|
+
}
|
|
80
|
+
/** An HTTP request captured by a Helios honeypot. */
|
|
81
|
+
export interface HeliosHTTPEvent {
|
|
82
|
+
/** When the request was captured, in Unix milliseconds. */
|
|
83
|
+
timestamp: UnixTime;
|
|
84
|
+
/**
|
|
85
|
+
* The proxy tunnel the request arrived through, as the exact decimal digits of an
|
|
86
|
+
* unsigned 64-bit integer.
|
|
87
|
+
*/
|
|
88
|
+
tunnel_id: string;
|
|
89
|
+
/** The destination host the tunnel was opened to. */
|
|
90
|
+
domain: string;
|
|
91
|
+
/** The destination port. */
|
|
92
|
+
port: number;
|
|
93
|
+
protocol: "http";
|
|
94
|
+
meta: HeliosMeta;
|
|
95
|
+
details?: HTTPRequestDetails;
|
|
96
|
+
/** The raw request bytes, base64-encoded. */
|
|
97
|
+
raw?: string;
|
|
98
|
+
}
|
|
99
|
+
export interface HTTPRequestDetails {
|
|
100
|
+
method?: string;
|
|
101
|
+
/** The request target. */
|
|
102
|
+
uri?: string;
|
|
103
|
+
/** The HTTP version, such as "HTTP/1.1". */
|
|
104
|
+
version?: string;
|
|
105
|
+
headers?: Record<string, string>;
|
|
106
|
+
}
|
|
107
|
+
/** A TLS ClientHello captured by a Helios honeypot. */
|
|
108
|
+
export interface HeliosHTTPSEvent {
|
|
109
|
+
/** When the handshake was captured, in Unix milliseconds. */
|
|
110
|
+
timestamp: UnixTime;
|
|
111
|
+
/**
|
|
112
|
+
* The proxy tunnel the handshake arrived through, as the exact decimal digits of an
|
|
113
|
+
* unsigned 64-bit integer.
|
|
114
|
+
*/
|
|
115
|
+
tunnel_id: string;
|
|
116
|
+
/** The destination host the tunnel was opened to. */
|
|
117
|
+
domain: string;
|
|
118
|
+
/** The destination port. */
|
|
119
|
+
port: number;
|
|
120
|
+
protocol: "https";
|
|
121
|
+
meta: HeliosMeta;
|
|
122
|
+
/** The parsed ClientHello, or null when the sensor could not parse the handshake. */
|
|
123
|
+
details?: TLSClientHello | null;
|
|
124
|
+
/** Always null: the HTTPS sensors do not capture handshake bytes. */
|
|
125
|
+
raw?: string | null;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* A TLS ClientHello as parsed by the Helios sensors. Fields the sensor could not parse
|
|
129
|
+
* are absent, and new fields may appear.
|
|
130
|
+
*/
|
|
131
|
+
export interface TLSClientHello {
|
|
132
|
+
/** The record layer version, such as "TLS 1.0". */
|
|
133
|
+
record_version?: string;
|
|
134
|
+
/** The handshake version, such as "TLS 1.2". */
|
|
135
|
+
handshake_version?: string;
|
|
136
|
+
/** The hex-encoded 32-byte client random. */
|
|
137
|
+
client_random?: string;
|
|
138
|
+
/** The hex-encoded session ID. */
|
|
139
|
+
session_id?: string;
|
|
140
|
+
/** The session ID length in bytes. */
|
|
141
|
+
session_id_length?: number;
|
|
142
|
+
cipher_suites?: TLSCodeName[];
|
|
143
|
+
compression_methods?: number[];
|
|
144
|
+
/** The extensions in wire order. */
|
|
145
|
+
extensions?: TLSExtension[];
|
|
146
|
+
/** The server_name extension value. */
|
|
147
|
+
sni?: string;
|
|
148
|
+
alpn?: string[];
|
|
149
|
+
supported_versions?: string[];
|
|
150
|
+
/** The supported key exchange groups. */
|
|
151
|
+
supported_groups?: TLSCodeName[];
|
|
152
|
+
signature_algorithms?: TLSCodeName[];
|
|
153
|
+
key_share_groups?: string[];
|
|
154
|
+
ec_point_formats?: string[];
|
|
155
|
+
psk_key_exchange_modes?: string[];
|
|
156
|
+
/** The supported certificate compression algorithms. */
|
|
157
|
+
compress_certificate?: string[];
|
|
158
|
+
has_grease?: boolean;
|
|
159
|
+
/** Whether the application_settings extension is present. */
|
|
160
|
+
application_settings?: boolean;
|
|
161
|
+
/** Whether the delegated_credentials extension is present. */
|
|
162
|
+
delegated_credentials?: boolean;
|
|
163
|
+
/** Whether the encrypt_then_mac extension is present. */
|
|
164
|
+
encrypt_then_mac?: boolean;
|
|
165
|
+
/** Whether the extended_master_secret extension is present. */
|
|
166
|
+
extended_master_secret?: boolean;
|
|
167
|
+
/** Whether the post_handshake_auth extension is present. */
|
|
168
|
+
post_handshake_auth?: boolean;
|
|
169
|
+
/** Whether the renegotiation_info extension is present. */
|
|
170
|
+
renegotiation_info?: boolean;
|
|
171
|
+
/** Whether the signed_certificate_timestamp extension is present. */
|
|
172
|
+
signed_certificate_timestamps?: boolean;
|
|
173
|
+
/** Whether the status_request extension is present. */
|
|
174
|
+
status_request?: boolean;
|
|
175
|
+
}
|
|
176
|
+
/** A TLS registry value, such as a cipher suite or group, with its name. */
|
|
177
|
+
export interface TLSCodeName {
|
|
178
|
+
code?: number;
|
|
179
|
+
name?: string;
|
|
180
|
+
}
|
|
181
|
+
export interface TLSExtension {
|
|
182
|
+
code?: number;
|
|
183
|
+
name?: string;
|
|
184
|
+
/** The extension data length in bytes. */
|
|
185
|
+
length?: number;
|
|
186
|
+
}
|
|
187
|
+
/** A DNS tunnel captured by a Helios honeypot. */
|
|
188
|
+
export interface HeliosDNSEvent {
|
|
189
|
+
/** When the tunnel was captured, in Unix milliseconds. */
|
|
190
|
+
timestamp: UnixTime;
|
|
191
|
+
/** The proxy tunnel, as the exact decimal digits of an unsigned 64-bit integer. */
|
|
192
|
+
tunnel_id: string;
|
|
193
|
+
/** The destination host the tunnel was opened to. */
|
|
194
|
+
domain: string;
|
|
195
|
+
/** The destination port. */
|
|
196
|
+
port: number;
|
|
197
|
+
protocol: "dns";
|
|
198
|
+
meta: HeliosMeta;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* An Android Debug Bridge shell command captured by a Helios honeypot. The sensor sends
|
|
202
|
+
* no timestamp.
|
|
203
|
+
*/
|
|
204
|
+
export interface HeliosADBEvent {
|
|
205
|
+
/** Identifies the ADB session, as 32 hex characters. */
|
|
206
|
+
session: string;
|
|
207
|
+
/** The 1-based position of the command within its session. */
|
|
208
|
+
sequential_id: number;
|
|
209
|
+
/** The shell command as plain text. Invalid UTF-8 bytes become U+FFFD. */
|
|
210
|
+
command: string;
|
|
211
|
+
/**
|
|
212
|
+
* The SHA-256 the sensor assigns to the session's payload, as 64 hex characters. It is
|
|
213
|
+
* not a hash of `command`; every command in a session shares it, as do sessions that
|
|
214
|
+
* replay the same script.
|
|
215
|
+
*/
|
|
216
|
+
hash: string;
|
|
217
|
+
}
|
package/dist/events.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/feed.d.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { type RequestOptions, type Transport } from "./transport.ts";
|
|
2
|
+
import type { Export, ExportMeta, ExportPage } from "./types.ts";
|
|
3
|
+
type Reviver = (key: string, value: unknown, context?: {
|
|
4
|
+
source: string;
|
|
5
|
+
}) => unknown;
|
|
6
|
+
/**
|
|
7
|
+
* Keeps each `tunnel_id` as the exact decimal digits sent on the wire, since these
|
|
8
|
+
* unsigned 64-bit IDs lose precision as numbers above 2^53.
|
|
9
|
+
*/
|
|
10
|
+
export declare const exactTunnelID: Reviver;
|
|
11
|
+
export interface ListExportsOptions extends RequestOptions {
|
|
12
|
+
/** The page size. The API default is 100, and it clamps values above 500. */
|
|
13
|
+
limit?: number | undefined;
|
|
14
|
+
/** Continues a listing from a previous page's `next_cursor`. */
|
|
15
|
+
cursor?: string | undefined;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Returns the ID of the daily export for the UTC day containing `date`. Daily exports
|
|
19
|
+
* roll up a day's hourly exports once the day ends.
|
|
20
|
+
*/
|
|
21
|
+
export declare function dailyExport(date: Date): string;
|
|
22
|
+
/**
|
|
23
|
+
* Returns the ID of the hourly export for the UTC hour containing `date`, with the hour
|
|
24
|
+
* unpadded as in `Export.id`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function hourlyExport(date: Date): string;
|
|
27
|
+
/**
|
|
28
|
+
* A Synthient data feed. Each feed has a real-time stream of events of type `T` and
|
|
29
|
+
* hourly parquet exports that roll up into daily exports. Access feeds through the
|
|
30
|
+
* client, such as `client.proxies`.
|
|
31
|
+
*
|
|
32
|
+
* Export IDs are "latest" for the most recent hourly export, a UTC day as
|
|
33
|
+
* "YYYY-MM-DD", or a UTC hour as "YYYY-MM-DD/H" (a zero-padded hour is also accepted).
|
|
34
|
+
* `Export.id` values and the results of {@link dailyExport} and {@link hourlyExport} can
|
|
35
|
+
* be passed as they are.
|
|
36
|
+
*/
|
|
37
|
+
export declare class Feed<T> {
|
|
38
|
+
#private;
|
|
39
|
+
/** @internal Feeds are created by the client. */
|
|
40
|
+
constructor(transport: Transport, path: string, reviver?: Reviver);
|
|
41
|
+
/**
|
|
42
|
+
* Streams the feed's real-time events. The connection opens on the first `next()`,
|
|
43
|
+
* which throws any connection error, such as an APIError with status 403 for a
|
|
44
|
+
* missing scope. The server closes each stream after up to 30 minutes, which ends the
|
|
45
|
+
* iteration normally; any other failure throws. Breaking out of the loop closes the
|
|
46
|
+
* connection. Events sent between connections are missed, so call `stream` again to
|
|
47
|
+
* reconnect. The client's `timeout` bounds the gap between data, and the server sends
|
|
48
|
+
* a keepalive about every 15 seconds.
|
|
49
|
+
*/
|
|
50
|
+
stream(options?: RequestOptions): AsyncGenerator<T, void, undefined>;
|
|
51
|
+
/** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
|
|
52
|
+
listExports(options?: ListExportsOptions): Promise<ExportPage>;
|
|
53
|
+
/**
|
|
54
|
+
* Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
|
|
55
|
+
* copied when `allExports` is called; errors, including invalid parameters, are
|
|
56
|
+
* thrown from the iteration.
|
|
57
|
+
*/
|
|
58
|
+
allExports({ limit, cursor, signal }?: ListExportsOptions): AsyncGenerator<Export, void, undefined>;
|
|
59
|
+
/** Returns the checksum, size, row count, and parquet schema of an export. */
|
|
60
|
+
exportMeta(id: string, options?: RequestOptions): Promise<ExportMeta>;
|
|
61
|
+
/**
|
|
62
|
+
* Returns a presigned URL for an export's parquet file, for tools that read parquet
|
|
63
|
+
* over HTTPS directly. The URL is valid for 24 hours and grants access without an API
|
|
64
|
+
* key, so treat it as a secret.
|
|
65
|
+
*/
|
|
66
|
+
downloadURL(id: string, options?: RequestOptions): Promise<URL>;
|
|
67
|
+
/**
|
|
68
|
+
* Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
|
|
69
|
+
* to release the connection. The API key is sent only to the Synthient API, never to
|
|
70
|
+
* the storage host that serves the file, and errors never include the presigned URL.
|
|
71
|
+
* A non-success response from either returns an APIError.
|
|
72
|
+
*/
|
|
73
|
+
download(id: string, options?: RequestOptions): Promise<ReadableStream<Uint8Array>>;
|
|
74
|
+
}
|
|
75
|
+
export {};
|
package/dist/feed.js
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { SynthientError, isObject } from "./errors.js";
|
|
2
|
+
import { limitQuery } from "./options.js";
|
|
3
|
+
import { drain, redirectTarget } from "./transport.js";
|
|
4
|
+
const MAX_EVENT_SIZE = 16 << 20;
|
|
5
|
+
const EXPORT_ID = /^(\d{4})-(\d{2})-(\d{2})(?:\/(\d{1,2}))?$/;
|
|
6
|
+
/**
|
|
7
|
+
* Keeps each `tunnel_id` as the exact decimal digits sent on the wire, since these
|
|
8
|
+
* unsigned 64-bit IDs lose precision as numbers above 2^53.
|
|
9
|
+
*/
|
|
10
|
+
export const exactTunnelID = (key, value, context) => key === "tunnel_id" && typeof value === "number" && context ? context.source : value;
|
|
11
|
+
/**
|
|
12
|
+
* Returns the ID of the daily export for the UTC day containing `date`. Daily exports
|
|
13
|
+
* roll up a day's hourly exports once the day ends.
|
|
14
|
+
*/
|
|
15
|
+
export function dailyExport(date) {
|
|
16
|
+
return date.toISOString().slice(0, 10);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Returns the ID of the hourly export for the UTC hour containing `date`, with the hour
|
|
20
|
+
* unpadded as in `Export.id`.
|
|
21
|
+
*/
|
|
22
|
+
export function hourlyExport(date) {
|
|
23
|
+
return `${dailyExport(date)}/${date.getUTCHours()}`;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* A Synthient data feed. Each feed has a real-time stream of events of type `T` and
|
|
27
|
+
* hourly parquet exports that roll up into daily exports. Access feeds through the
|
|
28
|
+
* client, such as `client.proxies`.
|
|
29
|
+
*
|
|
30
|
+
* Export IDs are "latest" for the most recent hourly export, a UTC day as
|
|
31
|
+
* "YYYY-MM-DD", or a UTC hour as "YYYY-MM-DD/H" (a zero-padded hour is also accepted).
|
|
32
|
+
* `Export.id` values and the results of {@link dailyExport} and {@link hourlyExport} can
|
|
33
|
+
* be passed as they are.
|
|
34
|
+
*/
|
|
35
|
+
export class Feed {
|
|
36
|
+
#transport;
|
|
37
|
+
#path;
|
|
38
|
+
#reviver;
|
|
39
|
+
/** @internal Feeds are created by the client. */
|
|
40
|
+
constructor(transport, path, reviver) {
|
|
41
|
+
this.#transport = transport;
|
|
42
|
+
this.#path = path;
|
|
43
|
+
this.#reviver = reviver;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Streams the feed's real-time events. The connection opens on the first `next()`,
|
|
47
|
+
* which throws any connection error, such as an APIError with status 403 for a
|
|
48
|
+
* missing scope. The server closes each stream after up to 30 minutes, which ends the
|
|
49
|
+
* iteration normally; any other failure throws. Breaking out of the loop closes the
|
|
50
|
+
* connection. Events sent between connections are missed, so call `stream` again to
|
|
51
|
+
* reconnect. The client's `timeout` bounds the gap between data, and the server sends
|
|
52
|
+
* a keepalive about every 15 seconds.
|
|
53
|
+
*/
|
|
54
|
+
async *stream(options) {
|
|
55
|
+
const signal = options?.signal;
|
|
56
|
+
const body = await this.#transport.stream({ path: `${this.#path}/stream`, accept: "application/x-ndjson", policy: "transient" }, signal);
|
|
57
|
+
yield* ndjson(body, signal, this.#reviver);
|
|
58
|
+
}
|
|
59
|
+
/** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
|
|
60
|
+
async listExports(options) {
|
|
61
|
+
const query = limitQuery(options?.limit);
|
|
62
|
+
if (options?.cursor) {
|
|
63
|
+
query.cursor = options.cursor;
|
|
64
|
+
}
|
|
65
|
+
const page = await this.#transport.json({ path: `${this.#path}/export`, query, policy: "transient" }, options?.signal);
|
|
66
|
+
const cursor = page.next_cursor;
|
|
67
|
+
if (!Array.isArray(page.feeds)) {
|
|
68
|
+
throw new SynthientError("export page has no feeds array");
|
|
69
|
+
}
|
|
70
|
+
if (cursor !== undefined && cursor !== null && typeof cursor !== "string") {
|
|
71
|
+
throw new SynthientError("export page has an invalid next_cursor");
|
|
72
|
+
}
|
|
73
|
+
return page;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
|
|
77
|
+
* copied when `allExports` is called; errors, including invalid parameters, are
|
|
78
|
+
* thrown from the iteration.
|
|
79
|
+
*/
|
|
80
|
+
async *allExports({ limit, cursor, signal } = {}) {
|
|
81
|
+
const seen = new Set([cursor ?? ""]);
|
|
82
|
+
for (;;) {
|
|
83
|
+
const page = await this.listExports({ limit, cursor, signal });
|
|
84
|
+
signal?.throwIfAborted();
|
|
85
|
+
for (const item of page.feeds) {
|
|
86
|
+
yield item;
|
|
87
|
+
signal?.throwIfAborted();
|
|
88
|
+
}
|
|
89
|
+
if (!page.next_cursor) {
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
if (seen.has(page.next_cursor)) {
|
|
93
|
+
throw new SynthientError("export listing returned a repeated cursor");
|
|
94
|
+
}
|
|
95
|
+
seen.add(page.next_cursor);
|
|
96
|
+
cursor = page.next_cursor;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/** Returns the checksum, size, row count, and parquet schema of an export. */
|
|
100
|
+
async exportMeta(id, options) {
|
|
101
|
+
return this.#transport.json({ path: `${this.#path}/export/${exportPath(id)}/meta`, policy: "transient" }, options?.signal);
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Returns a presigned URL for an export's parquet file, for tools that read parquet
|
|
105
|
+
* over HTTPS directly. The URL is valid for 24 hours and grants access without an API
|
|
106
|
+
* key, so treat it as a secret.
|
|
107
|
+
*/
|
|
108
|
+
async downloadURL(id, options) {
|
|
109
|
+
const request = {
|
|
110
|
+
path: `${this.#path}/export/${exportPath(id)}`,
|
|
111
|
+
policy: "transient",
|
|
112
|
+
};
|
|
113
|
+
const [response, attempt] = await this.#transport.send(request, options?.signal);
|
|
114
|
+
if (response.status >= 400) {
|
|
115
|
+
throw await this.#transport.error(response, attempt);
|
|
116
|
+
}
|
|
117
|
+
await drain(response, attempt);
|
|
118
|
+
if (response.status < 300) {
|
|
119
|
+
throw new SynthientError(`expected a redirect to the export, got status ${response.status}`);
|
|
120
|
+
}
|
|
121
|
+
return redirectTarget(new URL(this.#transport.url(request)), response);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
|
|
125
|
+
* to release the connection. The API key is sent only to the Synthient API, never to
|
|
126
|
+
* the storage host that serves the file, and errors never include the presigned URL.
|
|
127
|
+
* A non-success response from either returns an APIError.
|
|
128
|
+
*/
|
|
129
|
+
async download(id, options) {
|
|
130
|
+
const url = await this.downloadURL(id, options);
|
|
131
|
+
return this.#transport.download(url, options?.signal);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/** Converts an export ID to its URL path, with the hour unpadded. */
|
|
135
|
+
function exportPath(id) {
|
|
136
|
+
if (id === "latest") {
|
|
137
|
+
return id;
|
|
138
|
+
}
|
|
139
|
+
const match = EXPORT_ID.exec(id);
|
|
140
|
+
if (match) {
|
|
141
|
+
const [, year, month, day, hour] = match;
|
|
142
|
+
// An invalid day or an hour past 23 rolls the date over, so it no longer matches.
|
|
143
|
+
const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour ?? 0)));
|
|
144
|
+
if (dailyExport(date) === id.slice(0, 10)) {
|
|
145
|
+
return hour === undefined ? dailyExport(date) : hourlyExport(date);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
throw new TypeError("invalid export ID");
|
|
149
|
+
}
|
|
150
|
+
/** Blank lines are keepalives. A final line without a newline is still an event. */
|
|
151
|
+
async function* ndjson(body, signal, reviver) {
|
|
152
|
+
const reader = body.getReader();
|
|
153
|
+
const decoder = new TextDecoder();
|
|
154
|
+
let pending = "";
|
|
155
|
+
try {
|
|
156
|
+
for (;;) {
|
|
157
|
+
const { done, value } = await reader.read();
|
|
158
|
+
signal?.throwIfAborted();
|
|
159
|
+
const lines = decoder.decode(value, { stream: !done }).split("\n");
|
|
160
|
+
lines[0] = pending + (lines[0] ?? "");
|
|
161
|
+
pending = done ? "" : (lines.pop() ?? "");
|
|
162
|
+
if (Buffer.byteLength(pending) > MAX_EVENT_SIZE ||
|
|
163
|
+
lines.some((line) => Buffer.byteLength(line) > MAX_EVENT_SIZE)) {
|
|
164
|
+
throw new SynthientError(`stream event exceeds ${MAX_EVENT_SIZE} bytes`);
|
|
165
|
+
}
|
|
166
|
+
for (const line of lines) {
|
|
167
|
+
const text = line.trim();
|
|
168
|
+
if (text === "") {
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
let event;
|
|
172
|
+
try {
|
|
173
|
+
event = JSON.parse(text, reviver);
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
throw new SynthientError("invalid stream event", { cause: error });
|
|
177
|
+
}
|
|
178
|
+
if (!isObject(event)) {
|
|
179
|
+
throw new SynthientError("stream event is not a JSON object");
|
|
180
|
+
}
|
|
181
|
+
yield event;
|
|
182
|
+
signal?.throwIfAborted();
|
|
183
|
+
}
|
|
184
|
+
if (done) {
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
reader.cancel().catch(() => undefined);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { type FileDescriptorSet } from "@bufbuild/protobuf/wkt";
|
|
2
|
+
import { SynthientError } from "./errors.ts";
|
|
3
|
+
export declare const DEFAULT_ENDPOINT = "grpc.synthient.com:443";
|
|
4
|
+
export interface FetchSchemaOptions {
|
|
5
|
+
/** Sent as x-api-key metadata when not empty. Defaults to the SYNTHIENT_API_KEY environment variable. */
|
|
6
|
+
apiKey?: string | undefined;
|
|
7
|
+
/**
|
|
8
|
+
* The server as host:port, a bare host (port 443 is assumed), or a grpc:// or
|
|
9
|
+
* https:// URL. Defaults to {@link DEFAULT_ENDPOINT}.
|
|
10
|
+
*/
|
|
11
|
+
endpoint?: string | undefined;
|
|
12
|
+
/** Disables TLS, for local development servers. */
|
|
13
|
+
plaintext?: boolean | undefined;
|
|
14
|
+
/**
|
|
15
|
+
* Fully qualified service or message names to resolve. Empty resolves every listed
|
|
16
|
+
* service except the grpc.reflection services.
|
|
17
|
+
*/
|
|
18
|
+
symbols?: Iterable<string> | undefined;
|
|
19
|
+
/** Cancels the fetch, which rejects with the signal's reason. */
|
|
20
|
+
signal?: AbortSignal | undefined;
|
|
21
|
+
/** Milliseconds allowed for the whole fetch. Defaults to 30,000. */
|
|
22
|
+
timeout?: number | undefined;
|
|
23
|
+
}
|
|
24
|
+
export interface Schema {
|
|
25
|
+
/** The normalized address that was queried. */
|
|
26
|
+
endpoint: string;
|
|
27
|
+
/** The service or message names that were resolved. */
|
|
28
|
+
symbols: string[];
|
|
29
|
+
/** The resolved files in dependency order, each file after the files it imports. */
|
|
30
|
+
descriptorSet: FileDescriptorSet;
|
|
31
|
+
}
|
|
32
|
+
/** Fetching the schema failed. {@link explain} describes common causes. */
|
|
33
|
+
export declare class SchemaError extends SynthientError {
|
|
34
|
+
name: string;
|
|
35
|
+
/**
|
|
36
|
+
* The gRPC status code, as in `status` from `@grpc/grpc-js`, or undefined when the
|
|
37
|
+
* failure was not a gRPC status.
|
|
38
|
+
*/
|
|
39
|
+
readonly code: number | undefined;
|
|
40
|
+
constructor(message: string, code?: number, options?: ErrorOptions);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Fetches the protobuf files that define `options.symbols`, or every listed service,
|
|
44
|
+
* along with all of their imports.
|
|
45
|
+
*/
|
|
46
|
+
export declare function fetchSchema(options?: FetchSchemaOptions): Promise<Schema>;
|
|
47
|
+
/**
|
|
48
|
+
* Converts `raw`, in any form {@link FetchSchemaOptions.endpoint} accepts, to host:port.
|
|
49
|
+
* `host` is the bare host name, without brackets or a port.
|
|
50
|
+
*/
|
|
51
|
+
export declare function normalizeEndpoint(raw: string): {
|
|
52
|
+
endpoint: string;
|
|
53
|
+
host: string;
|
|
54
|
+
};
|
|
55
|
+
/** Returns a human-readable explanation of a common fetch error, or "". */
|
|
56
|
+
export declare function explain(error: unknown): string;
|