@synthient/synthient 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -24
- package/dist/client.d.ts +23 -7
- package/dist/client.js +41 -14
- package/dist/events.d.ts +26 -0
- package/dist/feed.d.ts +81 -10
- package/dist/feed.js +173 -52
- package/dist/grpcschema.d.ts +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.js +3 -3
- package/dist/types.d.ts +107 -14
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
# synthient-node
|
|
15
15
|
|
|
16
|
-
`@synthient/synthient` is the official Node.js SDK for the [Synthient API](https://docs.synthient.com): IP
|
|
16
|
+
`@synthient/synthient` is the official Node.js SDK for the [Synthient API](https://docs.synthient.com): IP intelligence, account details, and real-time and bulk data feeds. It is written in TypeScript and ships with type definitions.
|
|
17
17
|
|
|
18
18
|
[](https://www.npmjs.com/package/@synthient/synthient)
|
|
19
19
|
[](https://github.com/synthient/synthient-node/actions/workflows/test.yml)
|
|
@@ -83,7 +83,7 @@ Node.js 22.12 or later. Other runtimes, such as Bun and Deno, are not tested. Th
|
|
|
83
83
|
|
|
84
84
|
## API Keys
|
|
85
85
|
|
|
86
|
-
Every request needs a Synthient API key. Keys are issued from the Synthient dashboard; if you don't have one, [contact us](mailto:contact@synthient.com). A key's scopes decide what it can call: `BASIC` covers the lookups and `account`, and each feed has its own scopes, listed under [Feeds](#feeds).
|
|
86
|
+
Every request needs a Synthient API key. Keys are issued from the Synthient dashboard; if you don't have one, [contact us](mailto:contact@synthient.com). A key's scopes decide what it can call: `BASIC` covers the lookups and `account`, and each feed and dataset has its own scopes, listed under [Feeds](#feeds) and [Datasets](#datasets).
|
|
87
87
|
|
|
88
88
|
If a request fails even though you are sending a key:
|
|
89
89
|
|
|
@@ -138,7 +138,7 @@ if (info.intelligence) {
|
|
|
138
138
|
|
|
139
139
|
### Look up many IP addresses
|
|
140
140
|
|
|
141
|
-
Look up
|
|
141
|
+
Look up a batch of addresses in one request for `ceil(n * 0.9)` credits. The batch cap depends on your plan: 100 on Free, 500 on Starter, and 1000 on Premium and Enterprise, and the API rejects a larger batch with a 422. Private and reserved addresses are skipped and duplicates are merged, so match results by `ip`.
|
|
142
142
|
|
|
143
143
|
```ts
|
|
144
144
|
const results = await client.lookupIPs(["8.8.8.8", "1.1.1.1"]);
|
|
@@ -149,20 +149,6 @@ for (const result of results) {
|
|
|
149
149
|
}
|
|
150
150
|
```
|
|
151
151
|
|
|
152
|
-
### Look up a domain
|
|
153
|
-
|
|
154
|
-
Get the honeypot traffic observed for a domain and its most recent events. Each lookup costs one credit.
|
|
155
|
-
|
|
156
|
-
```ts
|
|
157
|
-
import { toDate } from "@synthient/synthient";
|
|
158
|
-
|
|
159
|
-
const domain = await client.lookupDomain("example.com", { limit: 10 });
|
|
160
|
-
console.log(domain.status, domain.stats.events_24h);
|
|
161
|
-
for (const event of domain.recent_events) {
|
|
162
|
-
console.log(toDate(event.timestamp), event.target_subdomain, event.port);
|
|
163
|
-
}
|
|
164
|
-
```
|
|
165
|
-
|
|
166
152
|
### Check your account
|
|
167
153
|
|
|
168
154
|
Get the key's account, scopes, and remaining lookup credits, at no cost.
|
|
@@ -180,8 +166,7 @@ console.log(account.organization.name, account.scopes, account.lookup_quota.cred
|
|
|
180
166
|
### Synthient
|
|
181
167
|
|
|
182
168
|
- `lookupIP` - Look up an IPv4 or IPv6 address
|
|
183
|
-
- `lookupIPs` - Look up
|
|
184
|
-
- `lookupDomain` - Look up the honeypot traffic observed for a domain
|
|
169
|
+
- `lookupIPs` - Look up a batch of addresses in one request
|
|
185
170
|
- `account` - Get the key's account, scopes, and remaining lookup credits
|
|
186
171
|
- `health` - Check that the API and its dependencies are healthy
|
|
187
172
|
|
|
@@ -192,6 +177,7 @@ Each feed is a property of the client, such as `client.proxies`, and has the sam
|
|
|
192
177
|
- `stream` - Iterate over the feed's real-time events
|
|
193
178
|
- `listExports` - List one page of the feed's Parquet exports, newest first
|
|
194
179
|
- `allExports` - Iterate over every export, fetching pages as needed
|
|
180
|
+
- `exportVolume` - Get the rows and bytes the feed exported per UTC day
|
|
195
181
|
- `exportMeta` - Get an export's checksum, size, row count, and Parquet schema
|
|
196
182
|
- `download` - Download an export's Parquet file
|
|
197
183
|
- `downloadURL` - Get a presigned URL for an export's Parquet file, valid for 24 hours
|
|
@@ -201,12 +187,27 @@ Each feed is a property of the client, such as `client.proxies`, and has the sam
|
|
|
201
187
|
| `proxies` | Residential, mobile, and datacenter proxies | `ProxyEvent` | `PROXY_FIREHOSE` | `PROXY_FEEDS` |
|
|
202
188
|
| `anonymizers` | VPN and anonymizer IP ranges | `AnonymizerEvent` | `ANONYMIZERS_STREAM` | `ANONYMIZERS_FEED` |
|
|
203
189
|
| `torrents` | BitTorrent DHT and tracker peers | `TorrentEvent` | `TORRENTS_STREAM` | `TORRENTS_FEED` |
|
|
204
|
-
| `ja4t` | JA4T TCP fingerprints | `JA4TEvent` | `JA4T_STREAM` | `JA4T_FEED` |
|
|
205
190
|
| `heliosHTTP` | Helios honeypot HTTP requests | `HeliosHTTPEvent` | `HONEYPOT_HTTP_STREAM` | `HONEYPOT_HTTP_FEED` |
|
|
206
191
|
| `heliosHTTPS` | Helios honeypot TLS ClientHellos | `HeliosHTTPSEvent` | `HONEYPOT_HTTPS_STREAM` | `HONEYPOT_HTTPS_FEED` |
|
|
207
192
|
| `heliosDNS` | Helios honeypot tunnel destinations | `HeliosDNSEvent` | `HONEYPOT_DNS_STREAM` | `HONEYPOT_DNS_FEED` |
|
|
208
193
|
| `heliosADB` | Helios honeypot Android Debug Bridge commands | `HeliosADBEvent` | `HONEYPOT_ADB_STREAM` | `HONEYPOT_ADB_FEED` |
|
|
209
194
|
|
|
195
|
+
### Datasets
|
|
196
|
+
|
|
197
|
+
Datasets have one export per UTC day and no stream. Each dataset is a property of the client, such as `client.geolocation`, and has the same methods:
|
|
198
|
+
|
|
199
|
+
- `listExports` - List one page of the dataset's daily exports, newest first
|
|
200
|
+
- `allExports` - Iterate over every export, fetching pages as needed
|
|
201
|
+
- `exportVolume` - Get the rows and bytes the dataset exported per UTC day
|
|
202
|
+
- `exportMeta` - Get a file's checksum, size, and row count, and list every file of the day
|
|
203
|
+
- `download` - Download a day's file in the format you choose
|
|
204
|
+
- `downloadURL` - Get a presigned URL for a day's file, valid for 24 hours
|
|
205
|
+
|
|
206
|
+
| Property | Data | Scope | Formats |
|
|
207
|
+
| ------------- | --------------------------------------- | ------------------ | ----------------------------------------- |
|
|
208
|
+
| `geolocation` | IP geolocation, network, and risk data | `GEOLOCATION_FEED` | `parquet` (default), `csv` (gzip), `mmdb` |
|
|
209
|
+
| `whois` | WHOIS registry records as Parquet files | `WHOIS_FEED` | `tar.gz` |
|
|
210
|
+
|
|
210
211
|
</details>
|
|
211
212
|
|
|
212
213
|
## Streaming
|
|
@@ -257,11 +258,19 @@ for await (const item of client.proxies.allExports()) {
|
|
|
257
258
|
}
|
|
258
259
|
```
|
|
259
260
|
|
|
261
|
+
Pass `day` to list only one UTC day's exports:
|
|
262
|
+
|
|
263
|
+
```ts
|
|
264
|
+
import { dailyExport } from "@synthient/synthient";
|
|
265
|
+
|
|
266
|
+
const today = await client.proxies.listExports({ day: dailyExport(new Date()) });
|
|
267
|
+
```
|
|
268
|
+
|
|
260
269
|
On the first page, the newest hourly export has the ID `latest`, a moving alias for the freshest export. Its `date` and `hour` give its stable ID, such as `2026-10-14/21`.
|
|
261
270
|
|
|
262
271
|
## Downloads
|
|
263
272
|
|
|
264
|
-
|
|
273
|
+
Feed exports are Parquet files. `download` returns a file's contents as a web `ReadableStream`, which you must read to the end or cancel:
|
|
265
274
|
|
|
266
275
|
```ts
|
|
267
276
|
import { createWriteStream } from "node:fs";
|
|
@@ -272,10 +281,19 @@ const body = await client.proxies.download("latest");
|
|
|
272
281
|
await pipeline(Readable.fromWeb(body), createWriteStream("proxies.parquet"));
|
|
273
282
|
```
|
|
274
283
|
|
|
275
|
-
|
|
284
|
+
Datasets offer more formats. This downloads the newest geolocation database as a MaxMind DB:
|
|
285
|
+
|
|
286
|
+
```ts
|
|
287
|
+
const mmdb = await client.geolocation.download("latest", { format: "mmdb" });
|
|
288
|
+
await pipeline(Readable.fromWeb(mmdb), createWriteStream("geolocation.mmdb"));
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
- Name an export by its `id` from a listing, by `"latest"`, or by `dailyExport(date)` or `hourlyExport(date)` for a UTC day or hour. Datasets have no hourly exports.
|
|
276
292
|
- `exportMeta` returns the file's SHA-256 checksum. Use a fixed ID for both calls, because `latest` can move between them.
|
|
277
293
|
- `downloadURL` returns a presigned URL that works without a key for 24 hours, so treat it as a secret. Your API key is never sent to the storage host.
|
|
278
294
|
- Hourly exports roll into the daily export shortly after 00:30 UTC the next day. After that, the hour returns a 400.
|
|
295
|
+
- `proxies` and `anonymizers` also publish lite exports, deduplicated to the latest row per address or range and provider. Pass `variant: "lite"` to `listExports` to list them. A lite export has the same `id` as the full one, so pass `variant: "lite"` again to `exportMeta`, `downloadURL`, and `download`.
|
|
296
|
+
- Files come back exactly as served: a `csv` download is a `.csv.gz` file, and a WHOIS download is a `.tar.gz` archive of Parquet files.
|
|
279
297
|
|
|
280
298
|
## Retries
|
|
281
299
|
|
|
@@ -370,7 +388,7 @@ try {
|
|
|
370
388
|
| 402 | The team is out of lookup credits |
|
|
371
389
|
| 403 | The key lacks the scope for the endpoint or feed |
|
|
372
390
|
| 404 | The export doesn't exist, or a daily export is still rolling up (`retryAfter` is set) |
|
|
373
|
-
| 422 | The input is invalid, such as a private address
|
|
391
|
+
| 422 | The input is invalid, such as a private address |
|
|
374
392
|
| 429 | Rate limited if `retryAfter` is set; otherwise the budget is spent or too many streams are open |
|
|
375
393
|
| 5xx | The server failed |
|
|
376
394
|
|
|
@@ -454,7 +472,7 @@ try {
|
|
|
454
472
|
}
|
|
455
473
|
```
|
|
456
474
|
|
|
457
|
-
`fetchSchema` reads
|
|
475
|
+
Server reflection requires an API key. `fetchSchema` reads it from `SYNTHIENT_API_KEY` unless you pass `apiKey`. `schema.descriptorSet` is a `FileDescriptorSet` from `@bufbuild/protobuf`. `explain` describes common gRPC errors in plain language.
|
|
458
476
|
|
|
459
477
|
# Development
|
|
460
478
|
|
package/dist/client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { AnonymizerEvent, HeliosADBEvent, HeliosDNSEvent, HeliosHTTPEvent, HeliosHTTPSEvent, JA4TEvent, ProxyEvent, TorrentEvent } from "./events.ts";
|
|
2
|
-
import { Feed } from "./feed.ts";
|
|
2
|
+
import { Dataset, Feed } from "./feed.ts";
|
|
3
3
|
import { type Fetch, type RequestOptions, type ResponseMetadata } from "./transport.ts";
|
|
4
4
|
import type { Account, DomainInfo, IPInfo } from "./types.ts";
|
|
5
5
|
export declare const DEFAULT_BASE_URL = "https://api.synthient.com/api/v4";
|
|
@@ -32,6 +32,10 @@ export interface ClientOptions {
|
|
|
32
32
|
*/
|
|
33
33
|
onResponse?: ((metadata: ResponseMetadata) => void) | undefined;
|
|
34
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* @deprecated Domain lookup was removed from the Synthient API and will stop working when
|
|
37
|
+
* the endpoint is turned off. It will be removed in the next major version.
|
|
38
|
+
*/
|
|
35
39
|
export interface LookupDomainOptions extends RequestOptions {
|
|
36
40
|
/** Caps the number of recent events returned. The API default is 50, and it clamps values above 1000. */
|
|
37
41
|
limit?: number | undefined;
|
|
@@ -66,6 +70,13 @@ export declare class Synthient {
|
|
|
66
70
|
readonly heliosDNS: Feed<HeliosDNSEvent>;
|
|
67
71
|
/** The Helios honeypot Android Debug Bridge command feed. Streaming needs the HONEYPOT_ADB_STREAM scope and exports need HONEYPOT_ADB_FEED. */
|
|
68
72
|
readonly heliosADB: Feed<HeliosADBEvent>;
|
|
73
|
+
/**
|
|
74
|
+
* The daily IP geolocation dataset, as "parquet", gzip-compressed "csv", and "mmdb"
|
|
75
|
+
* files. It needs the GEOLOCATION_FEED scope.
|
|
76
|
+
*/
|
|
77
|
+
readonly geolocation: Dataset;
|
|
78
|
+
/** The daily WHOIS registry dataset, as a "tar.gz" archive of parquet files. It needs the WHOIS_FEED scope. */
|
|
79
|
+
readonly whois: Dataset;
|
|
69
80
|
/** Throws a TypeError or RangeError if the API key is missing or an option is invalid. */
|
|
70
81
|
constructor(options?: ClientOptions);
|
|
71
82
|
/**
|
|
@@ -76,16 +87,21 @@ export declare class Synthient {
|
|
|
76
87
|
lookupIP(ip: string, options?: RequestOptions): Promise<IPInfo>;
|
|
77
88
|
/**
|
|
78
89
|
* Looks up as many as 1000 addresses in one request, billed as ceil(n × 0.9) credits
|
|
79
|
-
* for the n distinct usable addresses.
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
90
|
+
* for the n distinct usable addresses. The cap depends on the workspace plan: 100 on
|
|
91
|
+
* Free, 500 on Starter, and 1000 on Premium and Enterprise. Private and reserved
|
|
92
|
+
* addresses are skipped and duplicates collapse, so match results by their canonical
|
|
93
|
+
* `ip`, not by position. Results follow the order each address first appears. A batch
|
|
94
|
+
* over the plan's cap or with no usable address rejects with an APIError with status
|
|
95
|
+
* 422. An empty list returns [] without a request.
|
|
83
96
|
*/
|
|
84
97
|
lookupIPs(ips: Iterable<string>, options?: RequestOptions): Promise<IPInfo[]>;
|
|
85
98
|
/**
|
|
86
99
|
* Returns the honeypot traffic observed for a domain. A domain that was never observed
|
|
87
100
|
* resolves with status "inactive" and zeroed stats. Internationalized names must be
|
|
88
101
|
* sent in Punycode. Each lookup costs one credit.
|
|
102
|
+
*
|
|
103
|
+
* @deprecated Domain lookup was removed from the Synthient API and will stop working
|
|
104
|
+
* when the endpoint is turned off. It will be removed in the next major version.
|
|
89
105
|
*/
|
|
90
106
|
lookupDomain(domain: string, options?: LookupDomainOptions): Promise<DomainInfo>;
|
|
91
107
|
/**
|
|
@@ -94,8 +110,8 @@ export declare class Synthient {
|
|
|
94
110
|
*/
|
|
95
111
|
account(options?: RequestOptions): Promise<Account>;
|
|
96
112
|
/**
|
|
97
|
-
* Resolves if the API and its dependencies are healthy.
|
|
98
|
-
* once with an APIError with status 503.
|
|
113
|
+
* Resolves if the API and its dependencies are healthy. It needs a valid API key with
|
|
114
|
+
* any scope. An unhealthy API rejects at once with an APIError with status 503.
|
|
99
115
|
*/
|
|
100
116
|
health(options?: RequestOptions): Promise<void>;
|
|
101
117
|
}
|
package/dist/client.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import { SynthientError } from "./errors.js";
|
|
3
|
-
import { Feed, exactTunnelID } from "./feed.js";
|
|
3
|
+
import { Dataset, Feed, exactTunnelID } from "./feed.js";
|
|
4
4
|
import { checkTimeout, limitQuery, resolveAPIKey } from "./options.js";
|
|
5
5
|
import { Transport, isHTTP, kWait, wait, } from "./transport.js";
|
|
6
6
|
import { VERSION } from "./version.js";
|
|
@@ -34,6 +34,13 @@ export class Synthient {
|
|
|
34
34
|
heliosDNS;
|
|
35
35
|
/** The Helios honeypot Android Debug Bridge command feed. Streaming needs the HONEYPOT_ADB_STREAM scope and exports need HONEYPOT_ADB_FEED. */
|
|
36
36
|
heliosADB;
|
|
37
|
+
/**
|
|
38
|
+
* The daily IP geolocation dataset, as "parquet", gzip-compressed "csv", and "mmdb"
|
|
39
|
+
* files. It needs the GEOLOCATION_FEED scope.
|
|
40
|
+
*/
|
|
41
|
+
geolocation;
|
|
42
|
+
/** The daily WHOIS registry dataset, as a "tar.gz" archive of parquet files. It needs the WHOIS_FEED scope. */
|
|
43
|
+
whois;
|
|
37
44
|
#transport;
|
|
38
45
|
/** Throws a TypeError or RangeError if the API key is missing or an option is invalid. */
|
|
39
46
|
constructor(options = {}) {
|
|
@@ -71,10 +78,12 @@ export class Synthient {
|
|
|
71
78
|
this.anonymizers = new Feed(this.#transport, "/feeds/anonymizers");
|
|
72
79
|
this.torrents = new Feed(this.#transport, "/feeds/torrents");
|
|
73
80
|
this.ja4t = new Feed(this.#transport, "/feeds/ja4t");
|
|
74
|
-
this.heliosHTTP = new Feed(this.#transport, "/feeds/
|
|
75
|
-
this.heliosHTTPS = new Feed(this.#transport, "/feeds/
|
|
76
|
-
this.heliosDNS = new Feed(this.#transport, "/feeds/
|
|
77
|
-
this.heliosADB = new Feed(this.#transport, "/feeds/
|
|
81
|
+
this.heliosHTTP = new Feed(this.#transport, "/feeds/helios/http", exactTunnelID);
|
|
82
|
+
this.heliosHTTPS = new Feed(this.#transport, "/feeds/helios/https", exactTunnelID);
|
|
83
|
+
this.heliosDNS = new Feed(this.#transport, "/feeds/helios/dns", exactTunnelID);
|
|
84
|
+
this.heliosADB = new Feed(this.#transport, "/feeds/helios/adb");
|
|
85
|
+
this.geolocation = new Dataset(this.#transport, "/feeds/geolocation");
|
|
86
|
+
this.whois = new Dataset(this.#transport, "/feeds/whois");
|
|
78
87
|
}
|
|
79
88
|
/**
|
|
80
89
|
* Returns intelligence for an IPv4 or IPv6 address. Each lookup costs one credit;
|
|
@@ -85,14 +94,17 @@ export class Synthient {
|
|
|
85
94
|
if (typeof ip !== "string" || isIP(ip) === 0) {
|
|
86
95
|
throw new TypeError("invalid IP address");
|
|
87
96
|
}
|
|
88
|
-
|
|
97
|
+
const info = await this.#transport.json({ path: `/lookup/ip/${pathSegment(ip)}`, policy: "rateLimit" }, options?.signal);
|
|
98
|
+
return nullAsAbsent(info, "network", "location");
|
|
89
99
|
}
|
|
90
100
|
/**
|
|
91
101
|
* Looks up as many as 1000 addresses in one request, billed as ceil(n × 0.9) credits
|
|
92
|
-
* for the n distinct usable addresses.
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
102
|
+
* for the n distinct usable addresses. The cap depends on the workspace plan: 100 on
|
|
103
|
+
* Free, 500 on Starter, and 1000 on Premium and Enterprise. Private and reserved
|
|
104
|
+
* addresses are skipped and duplicates collapse, so match results by their canonical
|
|
105
|
+
* `ip`, not by position. Results follow the order each address first appears. A batch
|
|
106
|
+
* over the plan's cap or with no usable address rejects with an APIError with status
|
|
107
|
+
* 422. An empty list returns [] without a request.
|
|
96
108
|
*/
|
|
97
109
|
async lookupIPs(ips, options) {
|
|
98
110
|
const addresses = Array.from(ips);
|
|
@@ -114,13 +126,17 @@ export class Synthient {
|
|
|
114
126
|
if (!Array.isArray(batch.results)) {
|
|
115
127
|
throw new SynthientError("batch response has no results array");
|
|
116
128
|
}
|
|
117
|
-
return batch.results;
|
|
129
|
+
return batch.results.map((info) => nullAsAbsent(info, "network", "location"));
|
|
118
130
|
}
|
|
119
131
|
/**
|
|
120
132
|
* Returns the honeypot traffic observed for a domain. A domain that was never observed
|
|
121
133
|
* resolves with status "inactive" and zeroed stats. Internationalized names must be
|
|
122
134
|
* sent in Punycode. Each lookup costs one credit.
|
|
135
|
+
*
|
|
136
|
+
* @deprecated Domain lookup was removed from the Synthient API and will stop working
|
|
137
|
+
* when the endpoint is turned off. It will be removed in the next major version.
|
|
123
138
|
*/
|
|
139
|
+
// eslint-disable-next-line @typescript-eslint/no-deprecated -- its own deprecated types
|
|
124
140
|
async lookupDomain(domain, options) {
|
|
125
141
|
if (typeof domain !== "string" ||
|
|
126
142
|
domain === "" ||
|
|
@@ -140,16 +156,27 @@ export class Synthient {
|
|
|
140
156
|
* lookup credits. It costs no credits.
|
|
141
157
|
*/
|
|
142
158
|
async account(options) {
|
|
143
|
-
|
|
159
|
+
const account = await this.#transport.json({ path: "/account/me", policy: "transient" }, options?.signal);
|
|
160
|
+
return nullAsAbsent(account, "api_key");
|
|
144
161
|
}
|
|
145
162
|
/**
|
|
146
|
-
* Resolves if the API and its dependencies are healthy.
|
|
147
|
-
* once with an APIError with status 503.
|
|
163
|
+
* Resolves if the API and its dependencies are healthy. It needs a valid API key with
|
|
164
|
+
* any scope. An unhealthy API rejects at once with an APIError with status 503.
|
|
148
165
|
*/
|
|
149
166
|
async health(options) {
|
|
150
167
|
await this.#transport.json({ path: "/health", policy: "rateLimit" }, options?.signal);
|
|
151
168
|
}
|
|
152
169
|
}
|
|
170
|
+
// The API began sending null for some fields that these types model as absent when
|
|
171
|
+
// missing, so a null becomes absent to keep the types true.
|
|
172
|
+
function nullAsAbsent(value, ...keys) {
|
|
173
|
+
for (const key of keys) {
|
|
174
|
+
if (value[key] === null) {
|
|
175
|
+
Reflect.deleteProperty(value, key);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return value;
|
|
179
|
+
}
|
|
153
180
|
// Escapes a path segment like Go's url.PathEscape, which the other Synthient SDKs use:
|
|
154
181
|
// "/", "?", "#", and "%" are escaped and IPv6 colons are kept.
|
|
155
182
|
function pathSegment(value) {
|
package/dist/events.d.ts
CHANGED
|
@@ -66,6 +66,8 @@ export interface JA4TEvent {
|
|
|
66
66
|
asn?: number;
|
|
67
67
|
timestamp?: UnixTime;
|
|
68
68
|
}
|
|
69
|
+
/** The tunnel transport a Helios event arrived over. New values may be added. */
|
|
70
|
+
export type HeliosTransport = "tcp" | "udp" | "quic" | (string & {});
|
|
69
71
|
/** The Helios honeypot sensor that captured an event. */
|
|
70
72
|
export interface HeliosMeta {
|
|
71
73
|
/** The proxy exit address the traffic arrived through. */
|
|
@@ -119,6 +121,7 @@ export interface HeliosHTTPSEvent {
|
|
|
119
121
|
port: number;
|
|
120
122
|
protocol: "https";
|
|
121
123
|
meta: HeliosMeta;
|
|
124
|
+
transport: HeliosTransport;
|
|
122
125
|
/** The parsed ClientHello, or null when the sensor could not parse the handshake. */
|
|
123
126
|
details?: TLSClientHello | null;
|
|
124
127
|
/** Always null: the HTTPS sensors do not capture handshake bytes. */
|
|
@@ -172,6 +175,26 @@ export interface TLSClientHello {
|
|
|
172
175
|
signed_certificate_timestamps?: boolean;
|
|
173
176
|
/** Whether the status_request extension is present. */
|
|
174
177
|
status_request?: boolean;
|
|
178
|
+
/** False when the sensor captured only part of the ClientHello. */
|
|
179
|
+
complete?: boolean;
|
|
180
|
+
/** Present and true when the ClientHello was cut short. */
|
|
181
|
+
truncated?: boolean;
|
|
182
|
+
/** The transport the handshake was read from, as the sensor reports it. */
|
|
183
|
+
transport?: HeliosTransport;
|
|
184
|
+
/** `record_version` as its wire code point, such as 769. */
|
|
185
|
+
record_version_code?: number;
|
|
186
|
+
/** `handshake_version` as its wire code point, such as 771. */
|
|
187
|
+
handshake_version_code?: number;
|
|
188
|
+
/** The `supported_versions` entries as wire code points, in order, including GREASE values. */
|
|
189
|
+
supported_version_codes?: number[];
|
|
190
|
+
/** The `alpn` entries hex-encoded, in order. */
|
|
191
|
+
alpn_hex?: string[];
|
|
192
|
+
/** The TLS 1.3 signature_algorithms_cert extension. */
|
|
193
|
+
signature_algorithms_cert?: TLSCodeName[];
|
|
194
|
+
/** The record_size_limit extension value. */
|
|
195
|
+
record_size_limit?: number;
|
|
196
|
+
/** The max_fragment_length extension value. */
|
|
197
|
+
max_fragment_length?: number;
|
|
175
198
|
}
|
|
176
199
|
/** A TLS registry value, such as a cipher suite or group, with its name. */
|
|
177
200
|
export interface TLSCodeName {
|
|
@@ -196,6 +219,7 @@ export interface HeliosDNSEvent {
|
|
|
196
219
|
port: number;
|
|
197
220
|
protocol: "dns";
|
|
198
221
|
meta: HeliosMeta;
|
|
222
|
+
transport: HeliosTransport;
|
|
199
223
|
}
|
|
200
224
|
/**
|
|
201
225
|
* An Android Debug Bridge shell command captured by a Helios honeypot. The sensor sends
|
|
@@ -214,4 +238,6 @@ export interface HeliosADBEvent {
|
|
|
214
238
|
* replay the same script.
|
|
215
239
|
*/
|
|
216
240
|
hash: string;
|
|
241
|
+
/** Null from sensors that don't send it yet. */
|
|
242
|
+
meta: HeliosMeta | null;
|
|
217
243
|
}
|
package/dist/feed.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type RequestOptions, type Transport } from "./transport.ts";
|
|
2
|
-
import type { Export, ExportMeta, ExportPage } from "./types.ts";
|
|
2
|
+
import type { Export, ExportFormat, ExportMeta, ExportPage, ExportVariant, ExportVolume } from "./types.ts";
|
|
3
3
|
type Reviver = (key: string, value: unknown, context?: {
|
|
4
4
|
source: string;
|
|
5
5
|
}) => unknown;
|
|
@@ -13,6 +13,35 @@ export interface ListExportsOptions extends RequestOptions {
|
|
|
13
13
|
limit?: number | undefined;
|
|
14
14
|
/** Continues a listing from a previous page's `next_cursor`. */
|
|
15
15
|
cursor?: string | undefined;
|
|
16
|
+
/** Lists only the exports of this UTC day, as "YYYY-MM-DD" like {@link dailyExport} returns. */
|
|
17
|
+
day?: string | undefined;
|
|
18
|
+
/**
|
|
19
|
+
* Lists the lite files instead, which only the proxies, anonymizers, and JA4T feeds
|
|
20
|
+
* have; on other feeds the lite listing is empty. A lite export has the same `id` as
|
|
21
|
+
* the full one, so pass the variant again to download it.
|
|
22
|
+
*/
|
|
23
|
+
variant?: ExportVariant | undefined;
|
|
24
|
+
}
|
|
25
|
+
export interface ExportOptions extends RequestOptions {
|
|
26
|
+
/**
|
|
27
|
+
* Uses the export's lite file, which only the proxies, anonymizers, and JA4T feeds
|
|
28
|
+
* have. A lite file is published shortly after its export. Until then, and on other
|
|
29
|
+
* feeds, a lite export rejects with an APIError with status 404.
|
|
30
|
+
*/
|
|
31
|
+
variant?: ExportVariant | undefined;
|
|
32
|
+
}
|
|
33
|
+
export interface ExportVolumeOptions extends RequestOptions {
|
|
34
|
+
/** The number of UTC days to cover, including today. The API default is 90, and it accepts 1 to 366. */
|
|
35
|
+
days?: number | undefined;
|
|
36
|
+
}
|
|
37
|
+
export type ListDatasetExportsOptions = Omit<ListExportsOptions, "variant">;
|
|
38
|
+
export interface DatasetExportOptions extends RequestOptions {
|
|
39
|
+
/**
|
|
40
|
+
* The file to use. Geolocation has "parquet", the default, "csv", which is
|
|
41
|
+
* gzip-compressed with a header row, and "mmdb", a MaxMind DB. WHOIS has only
|
|
42
|
+
* "tar.gz". A format the day doesn't have rejects with an APIError with status 404.
|
|
43
|
+
*/
|
|
44
|
+
format?: ExportFormat | undefined;
|
|
16
45
|
}
|
|
17
46
|
/**
|
|
18
47
|
* Returns the ID of the daily export for the UTC day containing `date`. Daily exports
|
|
@@ -41,11 +70,11 @@ export declare class Feed<T> {
|
|
|
41
70
|
/**
|
|
42
71
|
* Streams the feed's real-time events. The connection opens on the first `next()`,
|
|
43
72
|
* which throws any connection error, such as an APIError with status 403 for a
|
|
44
|
-
* missing scope. The server
|
|
45
|
-
* iteration normally; any other failure throws. Breaking out of the
|
|
46
|
-
* connection. Events sent between connections are missed, so call
|
|
47
|
-
* reconnect. The client's `timeout` bounds the gap between data,
|
|
48
|
-
* a keepalive about every 15 seconds.
|
|
73
|
+
* missing scope. The server can close a stream at any time, such as during a deploy,
|
|
74
|
+
* which ends the iteration normally; any other failure throws. Breaking out of the
|
|
75
|
+
* loop closes the connection. Events sent between connections are missed, so call
|
|
76
|
+
* `stream` again to reconnect. The client's `timeout` bounds the gap between data,
|
|
77
|
+
* and the server sends a keepalive about every 15 seconds.
|
|
49
78
|
*/
|
|
50
79
|
stream(options?: RequestOptions): AsyncGenerator<T, void, undefined>;
|
|
51
80
|
/** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
|
|
@@ -55,21 +84,63 @@ export declare class Feed<T> {
|
|
|
55
84
|
* copied when `allExports` is called; errors, including invalid parameters, are
|
|
56
85
|
* thrown from the iteration.
|
|
57
86
|
*/
|
|
58
|
-
allExports(
|
|
87
|
+
allExports(options?: ListExportsOptions): AsyncGenerator<Export, void, undefined>;
|
|
88
|
+
/** Returns the rows and bytes the feed exported per UTC day, oldest first. */
|
|
89
|
+
exportVolume(options?: ExportVolumeOptions): Promise<ExportVolume>;
|
|
59
90
|
/** Returns the checksum, size, row count, and parquet schema of an export. */
|
|
60
|
-
exportMeta(id: string, options?:
|
|
91
|
+
exportMeta(id: string, options?: ExportOptions): Promise<ExportMeta>;
|
|
61
92
|
/**
|
|
62
93
|
* Returns a presigned URL for an export's parquet file, for tools that read parquet
|
|
63
94
|
* over HTTPS directly. The URL is valid for 24 hours and grants access without an API
|
|
64
95
|
* key, so treat it as a secret.
|
|
65
96
|
*/
|
|
66
|
-
downloadURL(id: string, options?:
|
|
97
|
+
downloadURL(id: string, options?: ExportOptions): Promise<URL>;
|
|
67
98
|
/**
|
|
68
99
|
* Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
|
|
69
100
|
* to release the connection. The API key is sent only to the Synthient API, never to
|
|
70
101
|
* the storage host that serves the file, and errors never include the presigned URL.
|
|
71
102
|
* A non-success response from either returns an APIError.
|
|
72
103
|
*/
|
|
73
|
-
download(id: string, options?:
|
|
104
|
+
download(id: string, options?: ExportOptions): Promise<ReadableStream<Uint8Array>>;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* A Synthient dataset, published as one export per UTC day with no stream and no hourly
|
|
108
|
+
* exports. Access datasets through the client, such as `client.geolocation`.
|
|
109
|
+
*
|
|
110
|
+
* Export IDs are "latest" for the newest day or a UTC day as "YYYY-MM-DD". `Export.id`
|
|
111
|
+
* values and the results of {@link dailyExport} can be passed as they are.
|
|
112
|
+
*/
|
|
113
|
+
export declare class Dataset {
|
|
114
|
+
#private;
|
|
115
|
+
/** @internal Datasets are created by the client. */
|
|
116
|
+
constructor(transport: Transport, path: string);
|
|
117
|
+
/** Returns one page of the dataset's exports, newest first. Use `allExports` to iterate every page. */
|
|
118
|
+
listExports(options?: ListDatasetExportsOptions): Promise<ExportPage>;
|
|
119
|
+
/**
|
|
120
|
+
* Iterates over the dataset's exports, newest first, fetching pages lazily. `options`
|
|
121
|
+
* is copied when `allExports` is called; errors, including invalid parameters, are
|
|
122
|
+
* thrown from the iteration.
|
|
123
|
+
*/
|
|
124
|
+
allExports(options?: ListDatasetExportsOptions): AsyncGenerator<Export, void, undefined>;
|
|
125
|
+
/** Returns the rows and bytes the dataset exported per UTC day, oldest first. */
|
|
126
|
+
exportVolume(options?: ExportVolumeOptions): Promise<ExportVolume>;
|
|
127
|
+
/**
|
|
128
|
+
* Returns the checksum, size, row count, and schema of an export's file in the
|
|
129
|
+
* requested format, and lists every file of the day.
|
|
130
|
+
*/
|
|
131
|
+
exportMeta(id: string, options?: DatasetExportOptions): Promise<ExportMeta>;
|
|
132
|
+
/**
|
|
133
|
+
* Returns a presigned URL for an export's file. The URL is valid for 24 hours and
|
|
134
|
+
* grants access without an API key, so treat it as a secret.
|
|
135
|
+
*/
|
|
136
|
+
downloadURL(id: string, options?: DatasetExportOptions): Promise<URL>;
|
|
137
|
+
/**
|
|
138
|
+
* Downloads an export's file as a byte stream, exactly as served: a "csv" file stays
|
|
139
|
+
* gzip-compressed and a WHOIS archive stays packed. Read it to the end or cancel it to
|
|
140
|
+
* release the connection. The API key is sent only to the Synthient API, never to the
|
|
141
|
+
* storage host that serves the file, and errors never include the presigned URL. A
|
|
142
|
+
* non-success response from either returns an APIError.
|
|
143
|
+
*/
|
|
144
|
+
download(id: string, options?: DatasetExportOptions): Promise<ReadableStream<Uint8Array>>;
|
|
74
145
|
}
|
|
75
146
|
export {};
|
package/dist/feed.js
CHANGED
|
@@ -36,20 +36,22 @@ export class Feed {
|
|
|
36
36
|
#transport;
|
|
37
37
|
#path;
|
|
38
38
|
#reviver;
|
|
39
|
+
#exports;
|
|
39
40
|
/** @internal Feeds are created by the client. */
|
|
40
41
|
constructor(transport, path, reviver) {
|
|
41
42
|
this.#transport = transport;
|
|
42
43
|
this.#path = path;
|
|
43
44
|
this.#reviver = reviver;
|
|
45
|
+
this.#exports = new Exports(transport, path, true);
|
|
44
46
|
}
|
|
45
47
|
/**
|
|
46
48
|
* Streams the feed's real-time events. The connection opens on the first `next()`,
|
|
47
49
|
* which throws any connection error, such as an APIError with status 403 for a
|
|
48
|
-
* missing scope. The server
|
|
49
|
-
* iteration normally; any other failure throws. Breaking out of the
|
|
50
|
-
* connection. Events sent between connections are missed, so call
|
|
51
|
-
* reconnect. The client's `timeout` bounds the gap between data,
|
|
52
|
-
* a keepalive about every 15 seconds.
|
|
50
|
+
* missing scope. The server can close a stream at any time, such as during a deploy,
|
|
51
|
+
* which ends the iteration normally; any other failure throws. Breaking out of the
|
|
52
|
+
* loop closes the connection. Events sent between connections are missed, so call
|
|
53
|
+
* `stream` again to reconnect. The client's `timeout` bounds the gap between data,
|
|
54
|
+
* and the server sends a keepalive about every 15 seconds.
|
|
53
55
|
*/
|
|
54
56
|
async *stream(options) {
|
|
55
57
|
const signal = options?.signal;
|
|
@@ -57,12 +59,119 @@ export class Feed {
|
|
|
57
59
|
yield* ndjson(body, signal, this.#reviver);
|
|
58
60
|
}
|
|
59
61
|
/** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
listExports(options) {
|
|
63
|
+
return this.#exports.list(options);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
|
|
67
|
+
* copied when `allExports` is called; errors, including invalid parameters, are
|
|
68
|
+
* thrown from the iteration.
|
|
69
|
+
*/
|
|
70
|
+
allExports(options) {
|
|
71
|
+
return this.#exports.all(options);
|
|
72
|
+
}
|
|
73
|
+
/** Returns the rows and bytes the feed exported per UTC day, oldest first. */
|
|
74
|
+
exportVolume(options) {
|
|
75
|
+
return this.#exports.volume(options);
|
|
76
|
+
}
|
|
77
|
+
/** Returns the checksum, size, row count, and parquet schema of an export. */
|
|
78
|
+
exportMeta(id, options) {
|
|
79
|
+
return this.#exports.meta(id, options);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Returns a presigned URL for an export's parquet file, for tools that read parquet
|
|
83
|
+
* over HTTPS directly. The URL is valid for 24 hours and grants access without an API
|
|
84
|
+
* key, so treat it as a secret.
|
|
85
|
+
*/
|
|
86
|
+
downloadURL(id, options) {
|
|
87
|
+
return this.#exports.downloadURL(id, options);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
|
|
91
|
+
* to release the connection. The API key is sent only to the Synthient API, never to
|
|
92
|
+
* the storage host that serves the file, and errors never include the presigned URL.
|
|
93
|
+
* A non-success response from either returns an APIError.
|
|
94
|
+
*/
|
|
95
|
+
download(id, options) {
|
|
96
|
+
return this.#exports.download(id, options);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* A Synthient dataset, published as one export per UTC day with no stream and no hourly
|
|
101
|
+
* exports. Access datasets through the client, such as `client.geolocation`.
|
|
102
|
+
*
|
|
103
|
+
* Export IDs are "latest" for the newest day or a UTC day as "YYYY-MM-DD". `Export.id`
|
|
104
|
+
* values and the results of {@link dailyExport} can be passed as they are.
|
|
105
|
+
*/
|
|
106
|
+
export class Dataset {
|
|
107
|
+
#exports;
|
|
108
|
+
/** @internal Datasets are created by the client. */
|
|
109
|
+
constructor(transport, path) {
|
|
110
|
+
this.#exports = new Exports(transport, path, false);
|
|
111
|
+
}
|
|
112
|
+
/** Returns one page of the dataset's exports, newest first. Use `allExports` to iterate every page. */
|
|
113
|
+
listExports(options) {
|
|
114
|
+
return this.#exports.list(options);
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Iterates over the dataset's exports, newest first, fetching pages lazily. `options`
|
|
118
|
+
* is copied when `allExports` is called; errors, including invalid parameters, are
|
|
119
|
+
* thrown from the iteration.
|
|
120
|
+
*/
|
|
121
|
+
allExports(options) {
|
|
122
|
+
return this.#exports.all(options);
|
|
123
|
+
}
|
|
124
|
+
/** Returns the rows and bytes the dataset exported per UTC day, oldest first. */
|
|
125
|
+
exportVolume(options) {
|
|
126
|
+
return this.#exports.volume(options);
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Returns the checksum, size, row count, and schema of an export's file in the
|
|
130
|
+
* requested format, and lists every file of the day.
|
|
131
|
+
*/
|
|
132
|
+
exportMeta(id, options) {
|
|
133
|
+
return this.#exports.meta(id, options);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Returns a presigned URL for an export's file. The URL is valid for 24 hours and
|
|
137
|
+
* grants access without an API key, so treat it as a secret.
|
|
138
|
+
*/
|
|
139
|
+
downloadURL(id, options) {
|
|
140
|
+
return this.#exports.downloadURL(id, options);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Downloads an export's file as a byte stream, exactly as served: a "csv" file stays
|
|
144
|
+
* gzip-compressed and a WHOIS archive stays packed. Read it to the end or cancel it to
|
|
145
|
+
* release the connection. The API key is sent only to the Synthient API, never to the
|
|
146
|
+
* storage host that serves the file, and errors never include the presigned URL. A
|
|
147
|
+
* non-success response from either returns an APIError.
|
|
148
|
+
*/
|
|
149
|
+
download(id, options) {
|
|
150
|
+
return this.#exports.download(id, options);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/** The export routes that feeds and datasets share under their path. */
|
|
154
|
+
class Exports {
|
|
155
|
+
#transport;
|
|
156
|
+
#path;
|
|
157
|
+
#hourly;
|
|
158
|
+
constructor(transport, path, hourly) {
|
|
159
|
+
this.#transport = transport;
|
|
160
|
+
this.#path = path;
|
|
161
|
+
this.#hourly = hourly;
|
|
162
|
+
}
|
|
163
|
+
async list(options = {}) {
|
|
164
|
+
const query = limitQuery(options.limit);
|
|
165
|
+
if (options.cursor) {
|
|
63
166
|
query.cursor = options.cursor;
|
|
64
167
|
}
|
|
65
|
-
|
|
168
|
+
if (options.day !== undefined) {
|
|
169
|
+
query.day = options.day;
|
|
170
|
+
}
|
|
171
|
+
if (options.variant !== undefined) {
|
|
172
|
+
query.variant = options.variant;
|
|
173
|
+
}
|
|
174
|
+
const page = await this.#transport.json({ path: `${this.#path}/export`, query, policy: "transient" }, options.signal);
|
|
66
175
|
const cursor = page.next_cursor;
|
|
67
176
|
if (!Array.isArray(page.feeds)) {
|
|
68
177
|
throw new SynthientError("export page has no feeds array");
|
|
@@ -72,19 +181,14 @@ export class Feed {
|
|
|
72
181
|
}
|
|
73
182
|
return page;
|
|
74
183
|
}
|
|
75
|
-
|
|
76
|
-
* Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
|
|
77
|
-
* copied when `allExports` is called; errors, including invalid parameters, are
|
|
78
|
-
* thrown from the iteration.
|
|
79
|
-
*/
|
|
80
|
-
async *allExports({ limit, cursor, signal } = {}) {
|
|
184
|
+
async *all({ cursor, ...options } = {}) {
|
|
81
185
|
const seen = new Set([cursor ?? ""]);
|
|
82
186
|
for (;;) {
|
|
83
|
-
const page = await this.
|
|
84
|
-
signal?.throwIfAborted();
|
|
187
|
+
const page = await this.list({ ...options, cursor });
|
|
188
|
+
options.signal?.throwIfAborted();
|
|
85
189
|
for (const item of page.feeds) {
|
|
86
190
|
yield item;
|
|
87
|
-
signal?.throwIfAborted();
|
|
191
|
+
options.signal?.throwIfAborted();
|
|
88
192
|
}
|
|
89
193
|
if (!page.next_cursor) {
|
|
90
194
|
return;
|
|
@@ -96,56 +200,73 @@ export class Feed {
|
|
|
96
200
|
cursor = page.next_cursor;
|
|
97
201
|
}
|
|
98
202
|
}
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
203
|
+
async volume(options = {}) {
|
|
204
|
+
const query = {};
|
|
205
|
+
if (options.days !== undefined) {
|
|
206
|
+
query.days = String(options.days);
|
|
207
|
+
}
|
|
208
|
+
return this.#transport.json({ path: `${this.#path}/export/daily`, query, policy: "transient" }, options.signal);
|
|
102
209
|
}
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
210
|
+
async meta(id, options = {}) {
|
|
211
|
+
const meta = await this.#transport.json({
|
|
212
|
+
path: `${this.#path}/export/${this.#exportPath(id)}/meta`,
|
|
213
|
+
query: fileQuery(options),
|
|
214
|
+
policy: "transient",
|
|
215
|
+
}, options.signal);
|
|
216
|
+
// The WHOIS meta once sent `"fields": null` for its archive's empty schema.
|
|
217
|
+
for (const { schema } of [meta, ...(meta.entities ?? [])]) {
|
|
218
|
+
if (schema) {
|
|
219
|
+
schema.fields = Array.isArray(schema.fields) ? schema.fields : [];
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
return meta;
|
|
223
|
+
}
|
|
224
|
+
async downloadURL(id, options = {}) {
|
|
109
225
|
const request = {
|
|
110
|
-
path: `${this.#path}/export/${exportPath(id)}`,
|
|
226
|
+
path: `${this.#path}/export/${this.#exportPath(id)}`,
|
|
227
|
+
query: fileQuery(options),
|
|
111
228
|
policy: "transient",
|
|
112
229
|
};
|
|
113
|
-
const [response, attempt] = await this.#transport.send(request, options
|
|
230
|
+
const [response, attempt] = await this.#transport.send(request, options.signal);
|
|
114
231
|
if (response.status >= 400) {
|
|
115
232
|
throw await this.#transport.error(response, attempt);
|
|
116
233
|
}
|
|
117
234
|
await drain(response, attempt);
|
|
118
|
-
if (response.status < 300) {
|
|
235
|
+
if (response.status < 300 || response.status === 304) {
|
|
119
236
|
throw new SynthientError(`expected a redirect to the export, got status ${response.status}`);
|
|
120
237
|
}
|
|
121
238
|
return redirectTarget(new URL(this.#transport.url(request)), response);
|
|
122
239
|
}
|
|
123
|
-
|
|
124
|
-
* Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
|
|
125
|
-
* to release the connection. The API key is sent only to the Synthient API, never to
|
|
126
|
-
* the storage host that serves the file, and errors never include the presigned URL.
|
|
127
|
-
* A non-success response from either returns an APIError.
|
|
128
|
-
*/
|
|
129
|
-
async download(id, options) {
|
|
240
|
+
async download(id, options = {}) {
|
|
130
241
|
const url = await this.downloadURL(id, options);
|
|
131
|
-
return this.#transport.download(url, options
|
|
242
|
+
return this.#transport.download(url, options.signal);
|
|
132
243
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
return id;
|
|
138
|
-
}
|
|
139
|
-
const match = EXPORT_ID.exec(id);
|
|
140
|
-
if (match) {
|
|
141
|
-
const [, year, month, day, hour] = match;
|
|
142
|
-
// An invalid day or an hour past 23 rolls the date over, so it no longer matches.
|
|
143
|
-
const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour ?? 0)));
|
|
144
|
-
if (dailyExport(date) === id.slice(0, 10)) {
|
|
145
|
-
return hour === undefined ? dailyExport(date) : hourlyExport(date);
|
|
244
|
+
/** Converts an export ID to its URL path, with the hour unpadded. */
|
|
245
|
+
#exportPath(id) {
|
|
246
|
+
if (id === "latest") {
|
|
247
|
+
return id;
|
|
146
248
|
}
|
|
249
|
+
const match = EXPORT_ID.exec(id);
|
|
250
|
+
if (match && (this.#hourly || match[4] === undefined)) {
|
|
251
|
+
const [, year, month, day, hour] = match;
|
|
252
|
+
// An invalid day or an hour past 23 rolls the date over, so it no longer matches.
|
|
253
|
+
const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour ?? 0)));
|
|
254
|
+
if (dailyExport(date) === id.slice(0, 10)) {
|
|
255
|
+
return hour === undefined ? dailyExport(date) : hourlyExport(date);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
throw new TypeError("invalid export ID");
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
function fileQuery({ variant, format }) {
|
|
262
|
+
const query = {};
|
|
263
|
+
if (variant !== undefined) {
|
|
264
|
+
query.variant = variant;
|
|
265
|
+
}
|
|
266
|
+
if (format !== undefined) {
|
|
267
|
+
query.format = format;
|
|
147
268
|
}
|
|
148
|
-
|
|
269
|
+
return query;
|
|
149
270
|
}
|
|
150
271
|
/** Blank lines are keepalives. A final line without a newline is still an event. */
|
|
151
272
|
async function* ndjson(body, signal, reviver) {
|
package/dist/grpcschema.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type FileDescriptorSet } from "@bufbuild/protobuf/wkt";
|
|
|
2
2
|
import { SynthientError } from "./errors.ts";
|
|
3
3
|
export declare const DEFAULT_ENDPOINT = "grpc.synthient.com:443";
|
|
4
4
|
export interface FetchSchemaOptions {
|
|
5
|
-
/** Sent as x-api-key metadata
|
|
5
|
+
/** Sent as x-api-key metadata, which server reflection requires. Defaults to the SYNTHIENT_API_KEY environment variable. */
|
|
6
6
|
apiKey?: string | undefined;
|
|
7
7
|
/**
|
|
8
8
|
* The server as host:port, a bare host (port 443 is assumed), or a grpc:// or
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The official Node.js client for the Synthient API: IP
|
|
3
|
-
*
|
|
2
|
+
* The official Node.js client for the Synthient API: IP intelligence, account details,
|
|
3
|
+
* and real-time and bulk data feeds.
|
|
4
4
|
*
|
|
5
5
|
* @example
|
|
6
6
|
* ```ts
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
export { DEFAULT_BASE_URL, Synthient, type ClientOptions, type LookupDomainOptions, } from "./client.ts";
|
|
17
17
|
export { APIConnectionError, APIError, APITimeoutError, SynthientError, type APIErrorInit, } from "./errors.ts";
|
|
18
18
|
export type * from "./events.ts";
|
|
19
|
-
export { Feed, dailyExport, hourlyExport, type ListExportsOptions } from "./feed.ts";
|
|
19
|
+
export { Dataset, Feed, dailyExport, hourlyExport, type DatasetExportOptions, type ExportOptions, type ExportVolumeOptions, type ListDatasetExportsOptions, type ListExportsOptions, } from "./feed.ts";
|
|
20
20
|
export type { Fetch, RequestOptions, ResponseMetadata } from "./transport.ts";
|
|
21
21
|
export { toDate } from "./types.ts";
|
|
22
22
|
export type * from "./types.ts";
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The official Node.js client for the Synthient API: IP
|
|
3
|
-
*
|
|
2
|
+
* The official Node.js client for the Synthient API: IP intelligence, account details,
|
|
3
|
+
* and real-time and bulk data feeds.
|
|
4
4
|
*
|
|
5
5
|
* @example
|
|
6
6
|
* ```ts
|
|
@@ -15,6 +15,6 @@
|
|
|
15
15
|
*/
|
|
16
16
|
export { DEFAULT_BASE_URL, Synthient, } from "./client.js";
|
|
17
17
|
export { APIConnectionError, APIError, APITimeoutError, SynthientError, } from "./errors.js";
|
|
18
|
-
export { Feed, dailyExport, hourlyExport } from "./feed.js";
|
|
18
|
+
export { Dataset, Feed, dailyExport, hourlyExport, } from "./feed.js";
|
|
19
19
|
export { toDate } from "./types.js";
|
|
20
20
|
export { VERSION } from "./version.js";
|
package/dist/types.d.ts
CHANGED
|
@@ -8,15 +8,28 @@ export type UnixTime = number;
|
|
|
8
8
|
export type NetworkType = "MOBILE" | "SATELLITE" | "IN_FLIGHT_WIFI" | "RESIDENTIAL" | "CORPORATE" | "ACADEMIC" | "DATACENTER" | "GOVERNMENT" | (string & {});
|
|
9
9
|
/** New values may be added. */
|
|
10
10
|
export type DeviceOS = "Android" | "Chrome OS" | "FreeBSD" | "iOS" | "Linux" | "Mac OS X" | "Windows" | (string & {});
|
|
11
|
-
/**
|
|
12
|
-
|
|
11
|
+
/**
|
|
12
|
+
* New values may be added. WHOIS_STREAM, NETWORK_IOCS_FEED, NETWORK_IOCS_STREAM,
|
|
13
|
+
* FILE_IOCS_FEED, and FILE_IOCS_STREAM are reserved for upcoming feeds, and HELIOS is a
|
|
14
|
+
* dashboard permission that authorizes no API operation. ADMIN_PROXIES is deprecated:
|
|
15
|
+
* it never authorized an API operation, the API no longer lists it, and it will be
|
|
16
|
+
* removed in the next major version.
|
|
17
|
+
*/
|
|
18
|
+
export type Scope = "BASIC" | "PROXY_FEEDS" | "PROXY_FIREHOSE" | "ANONYMIZERS_FEED" | "ANONYMIZERS_STREAM" | "JA3_FEED" | "JA3_STREAM" | "TORRENTS_FEED" | "TORRENTS_STREAM" | "HONEYPOT_HTTP_FEED" | "HONEYPOT_HTTP_STREAM" | "HONEYPOT_HTTPS_FEED" | "HONEYPOT_HTTPS_STREAM" | "HONEYPOT_ADB_FEED" | "HONEYPOT_ADB_STREAM" | "HONEYPOT_DNS_FEED" | "HONEYPOT_DNS_STREAM" | "JA4_FEED" | "JA4_STREAM" | "JA4T_FEED" | "JA4T_STREAM" | "HELIOS" | "WHOIS_FEED" | "WHOIS_STREAM" | "GEOLOCATION_FEED" | "NETWORK_IOCS_FEED" | "NETWORK_IOCS_STREAM" | "FILE_IOCS_FEED" | "FILE_IOCS_STREAM" | "ADMIN_PROXIES" | (string & {});
|
|
13
19
|
/** The API always sends one of these sentences. */
|
|
14
20
|
export type ActivityCadence = "No activity in the last 30 days." | "Activity is evenly distributed across the day." | "Activity follows a weekday business-hours pattern." | "Activity follows a weekend pattern." | "Activity is burst-driven with significant peaks.";
|
|
15
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* New values may be added. The datasets also send types such as "double", "date",
|
|
23
|
+
* "timestamp", and "list<string>".
|
|
24
|
+
*/
|
|
16
25
|
export type ParquetType = "string" | "int64" | "uint32" | "uint64" | "bool" | "bytes" | (string & {});
|
|
17
26
|
/** Converts a {@link UnixTime} to a Date, reading values of 1e12 or more as milliseconds. */
|
|
18
27
|
export declare function toDate(timestamp: UnixTime): Date;
|
|
19
|
-
/**
|
|
28
|
+
/**
|
|
29
|
+
* `network` is absent when the address lies in no known network range or the API's
|
|
30
|
+
* enrichment lookup failed, and `location` is absent along with it. `intelligence` is
|
|
31
|
+
* absent when the API has no data for it.
|
|
32
|
+
*/
|
|
20
33
|
export interface IPInfo {
|
|
21
34
|
ip: string;
|
|
22
35
|
network?: Network;
|
|
@@ -77,7 +90,12 @@ export interface Device {
|
|
|
77
90
|
version: string;
|
|
78
91
|
last_seen: UnixTime;
|
|
79
92
|
}
|
|
80
|
-
/**
|
|
93
|
+
/**
|
|
94
|
+
* The honeypot traffic Synthient has observed for a domain.
|
|
95
|
+
*
|
|
96
|
+
* @deprecated Domain lookup was removed from the Synthient API and will stop working when
|
|
97
|
+
* the endpoint is turned off. It will be removed in the next major version.
|
|
98
|
+
*/
|
|
81
99
|
export interface DomainInfo {
|
|
82
100
|
domain: string;
|
|
83
101
|
/**
|
|
@@ -189,6 +207,23 @@ export interface AccountAPIKey {
|
|
|
189
207
|
/** When the key was created, as an RFC 3339 string. */
|
|
190
208
|
created_at: string;
|
|
191
209
|
}
|
|
210
|
+
/** Whether an export covers a UTC day or a UTC hour. */
|
|
211
|
+
export type ExportKind = "hourly" | "daily";
|
|
212
|
+
/**
|
|
213
|
+
* The format of an export file. Feeds have only "parquet". The geolocation dataset adds
|
|
214
|
+
* "csv", which is gzip-compressed, and "mmdb", a MaxMind DB. The WHOIS dataset is one
|
|
215
|
+
* "tar.gz" archive of parquet files.
|
|
216
|
+
*/
|
|
217
|
+
export type ExportFormat = "parquet" | "csv" | "mmdb" | "tar.gz";
|
|
218
|
+
/**
|
|
219
|
+
* "full" is every sighting. "lite" is the snapshot deduplicated to the latest row in its
|
|
220
|
+
* hour or day, with the same columns: one row per address and provider for proxies, per
|
|
221
|
+
* address, fingerprint, and provider for JA4T, and per range and provider for
|
|
222
|
+
* anonymizers.
|
|
223
|
+
*/
|
|
224
|
+
export type ExportVariant = "full" | "lite";
|
|
225
|
+
/** A parquet file inside a WHOIS archive. */
|
|
226
|
+
export type WhoisEntity = "asns" | "networks" | "organisations" | "contacts" | "maintainers" | "transfers" | "delegations";
|
|
192
227
|
/** One page of a feed's exports, newest first. */
|
|
193
228
|
export interface ExportPage {
|
|
194
229
|
/** The feed's stream identifier, such as "proxies". */
|
|
@@ -198,7 +233,7 @@ export interface ExportPage {
|
|
|
198
233
|
/** Pass it as `cursor` to fetch the next page. It is absent, null, or "" on the last page. */
|
|
199
234
|
next_cursor?: string | null;
|
|
200
235
|
}
|
|
201
|
-
/**
|
|
236
|
+
/** An export of a feed or dataset. */
|
|
202
237
|
export interface Export {
|
|
203
238
|
/**
|
|
204
239
|
* Identifies the export: "YYYY-MM-DD" for a daily export, "YYYY-MM-DD/H" with an
|
|
@@ -206,37 +241,95 @@ export interface Export {
|
|
|
206
241
|
* first page. Pass it to `download`.
|
|
207
242
|
*/
|
|
208
243
|
id: string;
|
|
209
|
-
|
|
210
|
-
kind: "hourly" | "daily";
|
|
244
|
+
kind: ExportKind;
|
|
211
245
|
/** The UTC day of the export as "YYYY-MM-DD". */
|
|
212
246
|
date: string;
|
|
213
247
|
/** The UTC hour of an hourly export, absent or null for a daily export. */
|
|
214
248
|
hour?: number | null;
|
|
249
|
+
/** The size in bytes of the first file in `formats`. */
|
|
215
250
|
size_bytes: number;
|
|
251
|
+
/** The number of rows. For WHOIS, the sum of its entities' rows. */
|
|
216
252
|
row_count: number;
|
|
217
|
-
/** The hex-encoded SHA-256 of the
|
|
253
|
+
/** The hex-encoded SHA-256 of the first file in `formats`. */
|
|
218
254
|
checksum: string;
|
|
219
255
|
created_at: UnixTime;
|
|
220
256
|
/** The API path of the export's download, including the version prefix. */
|
|
221
257
|
download_path: string;
|
|
258
|
+
/** Every file of the export, its own format first. */
|
|
259
|
+
formats: ExportFile[];
|
|
260
|
+
/** The parquet files inside a WHOIS archive, absent for other exports. */
|
|
261
|
+
entities?: ExportEntity[];
|
|
262
|
+
/**
|
|
263
|
+
* "lite" in a lite listing, where the sizes, checksum, and `download_path` describe the
|
|
264
|
+
* lite file, and absent otherwise. `id` is the same for both variants, so pass the
|
|
265
|
+
* variant again to download the lite file.
|
|
266
|
+
*/
|
|
267
|
+
variant?: ExportVariant;
|
|
222
268
|
}
|
|
223
269
|
export interface ExportMeta {
|
|
224
270
|
id: string;
|
|
225
271
|
stream: string;
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
format:
|
|
272
|
+
kind: ExportKind;
|
|
273
|
+
/** The format of the file that `checksum`, `size`, and `schema` describe. */
|
|
274
|
+
format: ExportFormat;
|
|
229
275
|
/** The start of the export: midnight UTC for a daily export, the hour for an hourly one. */
|
|
230
276
|
date: UnixTime;
|
|
231
277
|
/** The UTC hour of an hourly export, absent or null for a daily export. */
|
|
232
278
|
hour?: number | null;
|
|
233
279
|
rows: number;
|
|
234
|
-
/** The size of the
|
|
280
|
+
/** The size of the file in bytes. */
|
|
235
281
|
size: number;
|
|
236
|
-
/** The hex-encoded SHA-256 of the
|
|
282
|
+
/** The hex-encoded SHA-256 of the file. */
|
|
237
283
|
checksum: string;
|
|
238
284
|
created_at: UnixTime;
|
|
285
|
+
/** The file's columns. A WHOIS archive has none; its parquet files list theirs in `entities`. */
|
|
239
286
|
schema: ParquetSchema;
|
|
287
|
+
/** Every file of the export, its own format first. */
|
|
288
|
+
formats: ExportFile[];
|
|
289
|
+
/** The parquet files inside a WHOIS archive, each with its columns, absent for other exports. */
|
|
290
|
+
entities?: ExportEntity[];
|
|
291
|
+
/**
|
|
292
|
+
* "lite" for a lite export, where the checksum, size, rows, schema, and formats describe
|
|
293
|
+
* the lite file, and absent otherwise.
|
|
294
|
+
*/
|
|
295
|
+
variant?: ExportVariant;
|
|
296
|
+
}
|
|
297
|
+
/** One file of an export. */
|
|
298
|
+
export interface ExportFile {
|
|
299
|
+
format: ExportFormat;
|
|
300
|
+
size_bytes: number;
|
|
301
|
+
/** The hex-encoded SHA-256 of the file as downloaded, so of the gzip file for "csv". */
|
|
302
|
+
checksum: string;
|
|
303
|
+
}
|
|
304
|
+
/** A parquet file inside a WHOIS archive. */
|
|
305
|
+
export interface ExportEntity {
|
|
306
|
+
entity: WhoisEntity;
|
|
307
|
+
row_count: number;
|
|
308
|
+
size_bytes: number;
|
|
309
|
+
/** The hex-encoded SHA-256 of the parquet file. */
|
|
310
|
+
checksum: string;
|
|
311
|
+
/** The parquet file's columns, present only in `exportMeta` results. */
|
|
312
|
+
schema?: ParquetSchema;
|
|
313
|
+
}
|
|
314
|
+
/** The rows and bytes a feed or dataset exported per UTC day. */
|
|
315
|
+
export interface ExportVolume {
|
|
316
|
+
/** The feed's stream identifier, such as "proxies". */
|
|
317
|
+
stream: string;
|
|
318
|
+
/** Oldest first. Today is included while in progress, and days without exports are left out. */
|
|
319
|
+
days: ExportVolumeDay[];
|
|
320
|
+
}
|
|
321
|
+
export interface ExportVolumeDay {
|
|
322
|
+
/** The UTC day as "YYYY-MM-DD". */
|
|
323
|
+
date: string;
|
|
324
|
+
/**
|
|
325
|
+
* "daily" when the day's rollup is counted. "hourly" when its hourly exports are summed
|
|
326
|
+
* instead, as for today or a day whose rollup is still pending.
|
|
327
|
+
*/
|
|
328
|
+
kind: ExportKind;
|
|
329
|
+
/** The number of files counted: 1 for a rollup, otherwise the hourly exports so far, of 24. */
|
|
330
|
+
snapshots: number;
|
|
331
|
+
row_count: number;
|
|
332
|
+
size_bytes: number;
|
|
240
333
|
}
|
|
241
334
|
export interface ParquetSchema {
|
|
242
335
|
/** The columns in file order. */
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** The client sends it in the User-Agent header. */
|
|
2
|
-
export declare const VERSION = "2.
|
|
2
|
+
export declare const VERSION = "2.1.0";
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** The client sends it in the User-Agent header. */
|
|
2
|
-
export const VERSION = "2.
|
|
2
|
+
export const VERSION = "2.1.0";
|