@synthient/synthient 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -13,7 +13,7 @@
13
13
 
14
14
  # synthient-node
15
15
 
16
- `@synthient/synthient` is the official Node.js SDK for the [Synthient API](https://docs.synthient.com): IP and domain intelligence, account details, and real-time and bulk data feeds. It is written in TypeScript and ships with type definitions.
16
+ `@synthient/synthient` is the official Node.js SDK for the [Synthient API](https://docs.synthient.com): IP intelligence, account details, and real-time and bulk data feeds. It is written in TypeScript and ships with type definitions.
17
17
 
18
18
  [![npm](https://img.shields.io/npm/v/@synthient/synthient)](https://www.npmjs.com/package/@synthient/synthient)
19
19
  [![test](https://github.com/synthient/synthient-node/actions/workflows/test.yml/badge.svg)](https://github.com/synthient/synthient-node/actions/workflows/test.yml)
@@ -83,7 +83,7 @@ Node.js 22.12 or later. Other runtimes, such as Bun and Deno, are not tested. Th
83
83
 
84
84
  ## API Keys
85
85
 
86
- Every request needs a Synthient API key. Keys are issued from the Synthient dashboard; if you don't have one, [contact us](mailto:contact@synthient.com). A key's scopes decide what it can call: `BASIC` covers the lookups and `account`, and each feed has its own scopes, listed under [Feeds](#feeds).
86
+ Every request needs a Synthient API key. Keys are issued from the Synthient dashboard; if you don't have one, [contact us](mailto:contact@synthient.com). A key's scopes decide what it can call: `BASIC` covers the lookups and `account`, and each feed and dataset has its own scopes, listed under [Feeds](#feeds) and [Datasets](#datasets).
87
87
 
88
88
  If a request fails even though you are sending a key:
89
89
 
@@ -138,7 +138,7 @@ if (info.intelligence) {
138
138
 
139
139
  ### Look up many IP addresses
140
140
 
141
- Look up as many as 1000 addresses in one request for `ceil(n * 0.9)` credits. Private and reserved addresses are skipped and duplicates are merged, so match results by `ip`.
141
+ Look up a batch of addresses in one request for `ceil(n * 0.9)` credits. The batch cap depends on your plan: 100 on Free, 500 on Starter, and 1000 on Premium and Enterprise, and the API rejects a larger batch with a 422. Private and reserved addresses are skipped and duplicates are merged, so match results by `ip`.
142
142
 
143
143
  ```ts
144
144
  const results = await client.lookupIPs(["8.8.8.8", "1.1.1.1"]);
@@ -149,20 +149,6 @@ for (const result of results) {
149
149
  }
150
150
  ```
151
151
 
152
- ### Look up a domain
153
-
154
- Get the honeypot traffic observed for a domain and its most recent events. Each lookup costs one credit.
155
-
156
- ```ts
157
- import { toDate } from "@synthient/synthient";
158
-
159
- const domain = await client.lookupDomain("example.com", { limit: 10 });
160
- console.log(domain.status, domain.stats.events_24h);
161
- for (const event of domain.recent_events) {
162
- console.log(toDate(event.timestamp), event.target_subdomain, event.port);
163
- }
164
- ```
165
-
166
152
  ### Check your account
167
153
 
168
154
  Get the key's account, scopes, and remaining lookup credits, at no cost.
@@ -180,8 +166,7 @@ console.log(account.organization.name, account.scopes, account.lookup_quota.cred
180
166
  ### Synthient
181
167
 
182
168
  - `lookupIP` - Look up an IPv4 or IPv6 address
183
- - `lookupIPs` - Look up as many as 1000 addresses in one request
184
- - `lookupDomain` - Look up the honeypot traffic observed for a domain
169
+ - `lookupIPs` - Look up a batch of addresses in one request
185
170
  - `account` - Get the key's account, scopes, and remaining lookup credits
186
171
  - `health` - Check that the API and its dependencies are healthy
187
172
 
@@ -192,6 +177,7 @@ Each feed is a property of the client, such as `client.proxies`, and has the sam
192
177
  - `stream` - Iterate over the feed's real-time events
193
178
  - `listExports` - List one page of the feed's Parquet exports, newest first
194
179
  - `allExports` - Iterate over every export, fetching pages as needed
180
+ - `exportVolume` - Get the rows and bytes the feed exported per UTC day
195
181
  - `exportMeta` - Get an export's checksum, size, row count, and Parquet schema
196
182
  - `download` - Download an export's Parquet file
197
183
  - `downloadURL` - Get a presigned URL for an export's Parquet file, valid for 24 hours
@@ -201,12 +187,27 @@ Each feed is a property of the client, such as `client.proxies`, and has the sam
201
187
  | `proxies` | Residential, mobile, and datacenter proxies | `ProxyEvent` | `PROXY_FIREHOSE` | `PROXY_FEEDS` |
202
188
  | `anonymizers` | VPN and anonymizer IP ranges | `AnonymizerEvent` | `ANONYMIZERS_STREAM` | `ANONYMIZERS_FEED` |
203
189
  | `torrents` | BitTorrent DHT and tracker peers | `TorrentEvent` | `TORRENTS_STREAM` | `TORRENTS_FEED` |
204
- | `ja4t` | JA4T TCP fingerprints | `JA4TEvent` | `JA4T_STREAM` | `JA4T_FEED` |
205
190
  | `heliosHTTP` | Helios honeypot HTTP requests | `HeliosHTTPEvent` | `HONEYPOT_HTTP_STREAM` | `HONEYPOT_HTTP_FEED` |
206
191
  | `heliosHTTPS` | Helios honeypot TLS ClientHellos | `HeliosHTTPSEvent` | `HONEYPOT_HTTPS_STREAM` | `HONEYPOT_HTTPS_FEED` |
207
192
  | `heliosDNS` | Helios honeypot tunnel destinations | `HeliosDNSEvent` | `HONEYPOT_DNS_STREAM` | `HONEYPOT_DNS_FEED` |
208
193
  | `heliosADB` | Helios honeypot Android Debug Bridge commands | `HeliosADBEvent` | `HONEYPOT_ADB_STREAM` | `HONEYPOT_ADB_FEED` |
209
194
 
195
+ ### Datasets
196
+
197
+ Datasets have one export per UTC day and no stream. Each dataset is a property of the client, such as `client.geolocation`, and has the same methods:
198
+
199
+ - `listExports` - List one page of the dataset's daily exports, newest first
200
+ - `allExports` - Iterate over every export, fetching pages as needed
201
+ - `exportVolume` - Get the rows and bytes the dataset exported per UTC day
202
+ - `exportMeta` - Get a file's checksum, size, and row count, and list every file of the day
203
+ - `download` - Download a day's file in the format you choose
204
+ - `downloadURL` - Get a presigned URL for a day's file, valid for 24 hours
205
+
206
+ | Property | Data | Scope | Formats |
207
+ | ------------- | --------------------------------------- | ------------------ | ----------------------------------------- |
208
+ | `geolocation` | IP geolocation, network, and risk data | `GEOLOCATION_FEED` | `parquet` (default), `csv` (gzip), `mmdb` |
209
+ | `whois` | WHOIS registry records as Parquet files | `WHOIS_FEED` | `tar.gz` |
210
+
210
211
  </details>
211
212
 
212
213
  ## Streaming
@@ -257,11 +258,19 @@ for await (const item of client.proxies.allExports()) {
257
258
  }
258
259
  ```
259
260
 
261
+ Pass `day` to list only one UTC day's exports:
262
+
263
+ ```ts
264
+ import { dailyExport } from "@synthient/synthient";
265
+
266
+ const today = await client.proxies.listExports({ day: dailyExport(new Date()) });
267
+ ```
268
+
260
269
  On the first page, the newest hourly export has the ID `latest`, a moving alias for the freshest export. Its `date` and `hour` give its stable ID, such as `2026-10-14/21`.
261
270
 
262
271
  ## Downloads
263
272
 
264
- Exports are Parquet files. `download` returns a file's contents as a web `ReadableStream`, which you must read to the end or cancel:
273
+ Feed exports are Parquet files. `download` returns a file's contents as a web `ReadableStream`, which you must read to the end or cancel:
265
274
 
266
275
  ```ts
267
276
  import { createWriteStream } from "node:fs";
@@ -272,10 +281,19 @@ const body = await client.proxies.download("latest");
272
281
  await pipeline(Readable.fromWeb(body), createWriteStream("proxies.parquet"));
273
282
  ```
274
283
 
275
- - Name an export by its `id` from a listing, by `"latest"`, or by `dailyExport(date)` or `hourlyExport(date)` for a UTC day or hour.
284
+ Datasets offer more formats. This downloads the newest geolocation database as a MaxMind DB:
285
+
286
+ ```ts
287
+ const mmdb = await client.geolocation.download("latest", { format: "mmdb" });
288
+ await pipeline(Readable.fromWeb(mmdb), createWriteStream("geolocation.mmdb"));
289
+ ```
290
+
291
+ - Name an export by its `id` from a listing, by `"latest"`, or by `dailyExport(date)` or `hourlyExport(date)` for a UTC day or hour. Datasets have no hourly exports.
276
292
  - `exportMeta` returns the file's SHA-256 checksum. Use a fixed ID for both calls, because `latest` can move between them.
277
293
  - `downloadURL` returns a presigned URL that works without a key for 24 hours, so treat it as a secret. Your API key is never sent to the storage host.
278
294
  - Hourly exports roll into the daily export shortly after 00:30 UTC the next day. After that, the hour returns a 400.
295
+ - `proxies` and `anonymizers` also publish lite exports, deduplicated to the latest row per address or range and provider. Pass `variant: "lite"` to `listExports` to list them. A lite export has the same `id` as the full one, so pass `variant: "lite"` again to `exportMeta`, `downloadURL`, and `download`.
296
+ - Files come back exactly as served: a `csv` download is a `.csv.gz` file, and a WHOIS download is a `.tar.gz` archive of Parquet files.
279
297
 
280
298
  ## Retries
281
299
 
@@ -370,7 +388,7 @@ try {
370
388
  | 402 | The team is out of lookup credits |
371
389
  | 403 | The key lacks the scope for the endpoint or feed |
372
390
  | 404 | The export doesn't exist, or a daily export is still rolling up (`retryAfter` is set) |
373
- | 422 | The input is invalid, such as a private address or a malformed domain |
391
+ | 422 | The input is invalid, such as a private address |
374
392
  | 429 | Rate limited if `retryAfter` is set; otherwise the budget is spent or too many streams are open |
375
393
  | 5xx | The server failed |
376
394
 
@@ -454,7 +472,7 @@ try {
454
472
  }
455
473
  ```
456
474
 
457
- `fetchSchema` reads the key from `SYNTHIENT_API_KEY` unless you pass `apiKey`. `schema.descriptorSet` is a `FileDescriptorSet` from `@bufbuild/protobuf`. `explain` describes common gRPC errors in plain language.
475
+ Server reflection requires an API key. `fetchSchema` reads it from `SYNTHIENT_API_KEY` unless you pass `apiKey`. `schema.descriptorSet` is a `FileDescriptorSet` from `@bufbuild/protobuf`. `explain` describes common gRPC errors in plain language.
458
476
 
459
477
  # Development
460
478
 
package/dist/client.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { AnonymizerEvent, HeliosADBEvent, HeliosDNSEvent, HeliosHTTPEvent, HeliosHTTPSEvent, JA4TEvent, ProxyEvent, TorrentEvent } from "./events.ts";
2
- import { Feed } from "./feed.ts";
2
+ import { Dataset, Feed } from "./feed.ts";
3
3
  import { type Fetch, type RequestOptions, type ResponseMetadata } from "./transport.ts";
4
4
  import type { Account, DomainInfo, IPInfo } from "./types.ts";
5
5
  export declare const DEFAULT_BASE_URL = "https://api.synthient.com/api/v4";
@@ -32,6 +32,10 @@ export interface ClientOptions {
32
32
  */
33
33
  onResponse?: ((metadata: ResponseMetadata) => void) | undefined;
34
34
  }
35
+ /**
36
+ * @deprecated Domain lookup was removed from the Synthient API and will stop working when
37
+ * the endpoint is turned off. It will be removed in the next major version.
38
+ */
35
39
  export interface LookupDomainOptions extends RequestOptions {
36
40
  /** Caps the number of recent events returned. The API default is 50, and it clamps values above 1000. */
37
41
  limit?: number | undefined;
@@ -66,6 +70,13 @@ export declare class Synthient {
66
70
  readonly heliosDNS: Feed<HeliosDNSEvent>;
67
71
  /** The Helios honeypot Android Debug Bridge command feed. Streaming needs the HONEYPOT_ADB_STREAM scope and exports need HONEYPOT_ADB_FEED. */
68
72
  readonly heliosADB: Feed<HeliosADBEvent>;
73
+ /**
74
+ * The daily IP geolocation dataset, as "parquet", gzip-compressed "csv", and "mmdb"
75
+ * files. It needs the GEOLOCATION_FEED scope.
76
+ */
77
+ readonly geolocation: Dataset;
78
+ /** The daily WHOIS registry dataset, as a "tar.gz" archive of parquet files. It needs the WHOIS_FEED scope. */
79
+ readonly whois: Dataset;
69
80
  /** Throws a TypeError or RangeError if the API key is missing or an option is invalid. */
70
81
  constructor(options?: ClientOptions);
71
82
  /**
@@ -76,16 +87,21 @@ export declare class Synthient {
76
87
  lookupIP(ip: string, options?: RequestOptions): Promise<IPInfo>;
77
88
  /**
78
89
  * Looks up as many as 1000 addresses in one request, billed as ceil(n × 0.9) credits
79
- * for the n distinct usable addresses. Private and reserved addresses are skipped and
80
- * duplicates collapse, so match results by their canonical `ip`, not by position.
81
- * Results follow the order each address first appears. A batch with no usable address
82
- * rejects with an APIError with status 422. An empty list returns [] without a request.
90
+ * for the n distinct usable addresses. The cap depends on the workspace plan: 100 on
91
+ * Free, 500 on Starter, and 1000 on Premium and Enterprise. Private and reserved
92
+ * addresses are skipped and duplicates collapse, so match results by their canonical
93
+ * `ip`, not by position. Results follow the order each address first appears. A batch
94
+ * over the plan's cap or with no usable address rejects with an APIError with status
95
+ * 422. An empty list returns [] without a request.
83
96
  */
84
97
  lookupIPs(ips: Iterable<string>, options?: RequestOptions): Promise<IPInfo[]>;
85
98
  /**
86
99
  * Returns the honeypot traffic observed for a domain. A domain that was never observed
87
100
  * resolves with status "inactive" and zeroed stats. Internationalized names must be
88
101
  * sent in Punycode. Each lookup costs one credit.
102
+ *
103
+ * @deprecated Domain lookup was removed from the Synthient API and will stop working
104
+ * when the endpoint is turned off. It will be removed in the next major version.
89
105
  */
90
106
  lookupDomain(domain: string, options?: LookupDomainOptions): Promise<DomainInfo>;
91
107
  /**
@@ -94,8 +110,8 @@ export declare class Synthient {
94
110
  */
95
111
  account(options?: RequestOptions): Promise<Account>;
96
112
  /**
97
- * Resolves if the API and its dependencies are healthy. An unhealthy API rejects at
98
- * once with an APIError with status 503.
113
+ * Resolves if the API and its dependencies are healthy. It needs a valid API key with
114
+ * any scope. An unhealthy API rejects at once with an APIError with status 503.
99
115
  */
100
116
  health(options?: RequestOptions): Promise<void>;
101
117
  }
package/dist/client.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { isIP } from "node:net";
2
2
  import { SynthientError } from "./errors.js";
3
- import { Feed, exactTunnelID } from "./feed.js";
3
+ import { Dataset, Feed, exactTunnelID } from "./feed.js";
4
4
  import { checkTimeout, limitQuery, resolveAPIKey } from "./options.js";
5
5
  import { Transport, isHTTP, kWait, wait, } from "./transport.js";
6
6
  import { VERSION } from "./version.js";
@@ -34,6 +34,13 @@ export class Synthient {
34
34
  heliosDNS;
35
35
  /** The Helios honeypot Android Debug Bridge command feed. Streaming needs the HONEYPOT_ADB_STREAM scope and exports need HONEYPOT_ADB_FEED. */
36
36
  heliosADB;
37
+ /**
38
+ * The daily IP geolocation dataset, as "parquet", gzip-compressed "csv", and "mmdb"
39
+ * files. It needs the GEOLOCATION_FEED scope.
40
+ */
41
+ geolocation;
42
+ /** The daily WHOIS registry dataset, as a "tar.gz" archive of parquet files. It needs the WHOIS_FEED scope. */
43
+ whois;
37
44
  #transport;
38
45
  /** Throws a TypeError or RangeError if the API key is missing or an option is invalid. */
39
46
  constructor(options = {}) {
@@ -71,10 +78,12 @@ export class Synthient {
71
78
  this.anonymizers = new Feed(this.#transport, "/feeds/anonymizers");
72
79
  this.torrents = new Feed(this.#transport, "/feeds/torrents");
73
80
  this.ja4t = new Feed(this.#transport, "/feeds/ja4t");
74
- this.heliosHTTP = new Feed(this.#transport, "/feeds/helio/http", exactTunnelID);
75
- this.heliosHTTPS = new Feed(this.#transport, "/feeds/helio/https", exactTunnelID);
76
- this.heliosDNS = new Feed(this.#transport, "/feeds/helio/dns", exactTunnelID);
77
- this.heliosADB = new Feed(this.#transport, "/feeds/helio/adb");
81
+ this.heliosHTTP = new Feed(this.#transport, "/feeds/helios/http", exactTunnelID);
82
+ this.heliosHTTPS = new Feed(this.#transport, "/feeds/helios/https", exactTunnelID);
83
+ this.heliosDNS = new Feed(this.#transport, "/feeds/helios/dns", exactTunnelID);
84
+ this.heliosADB = new Feed(this.#transport, "/feeds/helios/adb");
85
+ this.geolocation = new Dataset(this.#transport, "/feeds/geolocation");
86
+ this.whois = new Dataset(this.#transport, "/feeds/whois");
78
87
  }
79
88
  /**
80
89
  * Returns intelligence for an IPv4 or IPv6 address. Each lookup costs one credit;
@@ -85,14 +94,17 @@ export class Synthient {
85
94
  if (typeof ip !== "string" || isIP(ip) === 0) {
86
95
  throw new TypeError("invalid IP address");
87
96
  }
88
- return this.#transport.json({ path: `/lookup/ip/${pathSegment(ip)}`, policy: "rateLimit" }, options?.signal);
97
+ const info = await this.#transport.json({ path: `/lookup/ip/${pathSegment(ip)}`, policy: "rateLimit" }, options?.signal);
98
+ return nullAsAbsent(info, "network", "location");
89
99
  }
90
100
  /**
91
101
  * Looks up as many as 1000 addresses in one request, billed as ceil(n × 0.9) credits
92
- * for the n distinct usable addresses. Private and reserved addresses are skipped and
93
- * duplicates collapse, so match results by their canonical `ip`, not by position.
94
- * Results follow the order each address first appears. A batch with no usable address
95
- * rejects with an APIError with status 422. An empty list returns [] without a request.
102
+ * for the n distinct usable addresses. The cap depends on the workspace plan: 100 on
103
+ * Free, 500 on Starter, and 1000 on Premium and Enterprise. Private and reserved
104
+ * addresses are skipped and duplicates collapse, so match results by their canonical
105
+ * `ip`, not by position. Results follow the order each address first appears. A batch
106
+ * over the plan's cap or with no usable address rejects with an APIError with status
107
+ * 422. An empty list returns [] without a request.
96
108
  */
97
109
  async lookupIPs(ips, options) {
98
110
  const addresses = Array.from(ips);
@@ -114,13 +126,17 @@ export class Synthient {
114
126
  if (!Array.isArray(batch.results)) {
115
127
  throw new SynthientError("batch response has no results array");
116
128
  }
117
- return batch.results;
129
+ return batch.results.map((info) => nullAsAbsent(info, "network", "location"));
118
130
  }
119
131
  /**
120
132
  * Returns the honeypot traffic observed for a domain. A domain that was never observed
121
133
  * resolves with status "inactive" and zeroed stats. Internationalized names must be
122
134
  * sent in Punycode. Each lookup costs one credit.
135
+ *
136
+ * @deprecated Domain lookup was removed from the Synthient API and will stop working
137
+ * when the endpoint is turned off. It will be removed in the next major version.
123
138
  */
139
+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- its own deprecated types
124
140
  async lookupDomain(domain, options) {
125
141
  if (typeof domain !== "string" ||
126
142
  domain === "" ||
@@ -140,16 +156,27 @@ export class Synthient {
140
156
  * lookup credits. It costs no credits.
141
157
  */
142
158
  async account(options) {
143
- return this.#transport.json({ path: "/account/me", policy: "transient" }, options?.signal);
159
+ const account = await this.#transport.json({ path: "/account/me", policy: "transient" }, options?.signal);
160
+ return nullAsAbsent(account, "api_key");
144
161
  }
145
162
  /**
146
- * Resolves if the API and its dependencies are healthy. An unhealthy API rejects at
147
- * once with an APIError with status 503.
163
+ * Resolves if the API and its dependencies are healthy. It needs a valid API key with
164
+ * any scope. An unhealthy API rejects at once with an APIError with status 503.
148
165
  */
149
166
  async health(options) {
150
167
  await this.#transport.json({ path: "/health", policy: "rateLimit" }, options?.signal);
151
168
  }
152
169
  }
170
+ // The API began sending null for some fields that these types model as absent when
171
+ // missing, so a null becomes absent to keep the types true.
172
+ function nullAsAbsent(value, ...keys) {
173
+ for (const key of keys) {
174
+ if (value[key] === null) {
175
+ Reflect.deleteProperty(value, key);
176
+ }
177
+ }
178
+ return value;
179
+ }
153
180
  // Escapes a path segment like Go's url.PathEscape, which the other Synthient SDKs use:
154
181
  // "/", "?", "#", and "%" are escaped and IPv6 colons are kept.
155
182
  function pathSegment(value) {
package/dist/events.d.ts CHANGED
@@ -66,6 +66,8 @@ export interface JA4TEvent {
66
66
  asn?: number;
67
67
  timestamp?: UnixTime;
68
68
  }
69
+ /** The tunnel transport a Helios event arrived over. New values may be added. */
70
+ export type HeliosTransport = "tcp" | "udp" | "quic" | (string & {});
69
71
  /** The Helios honeypot sensor that captured an event. */
70
72
  export interface HeliosMeta {
71
73
  /** The proxy exit address the traffic arrived through. */
@@ -119,6 +121,7 @@ export interface HeliosHTTPSEvent {
119
121
  port: number;
120
122
  protocol: "https";
121
123
  meta: HeliosMeta;
124
+ transport: HeliosTransport;
122
125
  /** The parsed ClientHello, or null when the sensor could not parse the handshake. */
123
126
  details?: TLSClientHello | null;
124
127
  /** Always null: the HTTPS sensors do not capture handshake bytes. */
@@ -172,6 +175,26 @@ export interface TLSClientHello {
172
175
  signed_certificate_timestamps?: boolean;
173
176
  /** Whether the status_request extension is present. */
174
177
  status_request?: boolean;
178
+ /** False when the sensor captured only part of the ClientHello. */
179
+ complete?: boolean;
180
+ /** Present and true when the ClientHello was cut short. */
181
+ truncated?: boolean;
182
+ /** The transport the handshake was read from, as the sensor reports it. */
183
+ transport?: HeliosTransport;
184
+ /** `record_version` as its wire code point, such as 769. */
185
+ record_version_code?: number;
186
+ /** `handshake_version` as its wire code point, such as 771. */
187
+ handshake_version_code?: number;
188
+ /** The `supported_versions` entries as wire code points, in order, including GREASE values. */
189
+ supported_version_codes?: number[];
190
+ /** The `alpn` entries hex-encoded, in order. */
191
+ alpn_hex?: string[];
192
+ /** The TLS 1.3 signature_algorithms_cert extension. */
193
+ signature_algorithms_cert?: TLSCodeName[];
194
+ /** The record_size_limit extension value. */
195
+ record_size_limit?: number;
196
+ /** The max_fragment_length extension value. */
197
+ max_fragment_length?: number;
175
198
  }
176
199
  /** A TLS registry value, such as a cipher suite or group, with its name. */
177
200
  export interface TLSCodeName {
@@ -196,6 +219,7 @@ export interface HeliosDNSEvent {
196
219
  port: number;
197
220
  protocol: "dns";
198
221
  meta: HeliosMeta;
222
+ transport: HeliosTransport;
199
223
  }
200
224
  /**
201
225
  * An Android Debug Bridge shell command captured by a Helios honeypot. The sensor sends
@@ -214,4 +238,6 @@ export interface HeliosADBEvent {
214
238
  * replay the same script.
215
239
  */
216
240
  hash: string;
241
+ /** Null from sensors that don't send it yet. */
242
+ meta: HeliosMeta | null;
217
243
  }
package/dist/feed.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { type RequestOptions, type Transport } from "./transport.ts";
2
- import type { Export, ExportMeta, ExportPage } from "./types.ts";
2
+ import type { Export, ExportFormat, ExportMeta, ExportPage, ExportVariant, ExportVolume } from "./types.ts";
3
3
  type Reviver = (key: string, value: unknown, context?: {
4
4
  source: string;
5
5
  }) => unknown;
@@ -13,6 +13,35 @@ export interface ListExportsOptions extends RequestOptions {
13
13
  limit?: number | undefined;
14
14
  /** Continues a listing from a previous page's `next_cursor`. */
15
15
  cursor?: string | undefined;
16
+ /** Lists only the exports of this UTC day, as "YYYY-MM-DD" like {@link dailyExport} returns. */
17
+ day?: string | undefined;
18
+ /**
19
+ * Lists the lite files instead, which only the proxies, anonymizers, and JA4T feeds
20
+ * have; on other feeds the lite listing is empty. A lite export has the same `id` as
21
+ * the full one, so pass the variant again to download it.
22
+ */
23
+ variant?: ExportVariant | undefined;
24
+ }
25
+ export interface ExportOptions extends RequestOptions {
26
+ /**
27
+ * Uses the export's lite file, which only the proxies, anonymizers, and JA4T feeds
28
+ * have. A lite file is published shortly after its export. Until then, and on other
29
+ * feeds, a lite export rejects with an APIError with status 404.
30
+ */
31
+ variant?: ExportVariant | undefined;
32
+ }
33
+ export interface ExportVolumeOptions extends RequestOptions {
34
+ /** The number of UTC days to cover, including today. The API default is 90, and it accepts 1 to 366. */
35
+ days?: number | undefined;
36
+ }
37
+ export type ListDatasetExportsOptions = Omit<ListExportsOptions, "variant">;
38
+ export interface DatasetExportOptions extends RequestOptions {
39
+ /**
40
+ * The file to use. Geolocation has "parquet", the default, "csv", which is
41
+ * gzip-compressed with a header row, and "mmdb", a MaxMind DB. WHOIS has only
42
+ * "tar.gz". A format the day doesn't have rejects with an APIError with status 404.
43
+ */
44
+ format?: ExportFormat | undefined;
16
45
  }
17
46
  /**
18
47
  * Returns the ID of the daily export for the UTC day containing `date`. Daily exports
@@ -41,11 +70,11 @@ export declare class Feed<T> {
41
70
  /**
42
71
  * Streams the feed's real-time events. The connection opens on the first `next()`,
43
72
  * which throws any connection error, such as an APIError with status 403 for a
44
- * missing scope. The server closes each stream after up to 30 minutes, which ends the
45
- * iteration normally; any other failure throws. Breaking out of the loop closes the
46
- * connection. Events sent between connections are missed, so call `stream` again to
47
- * reconnect. The client's `timeout` bounds the gap between data, and the server sends
48
- * a keepalive about every 15 seconds.
73
+ * missing scope. The server can close a stream at any time, such as during a deploy,
74
+ * which ends the iteration normally; any other failure throws. Breaking out of the
75
+ * loop closes the connection. Events sent between connections are missed, so call
76
+ * `stream` again to reconnect. The client's `timeout` bounds the gap between data,
77
+ * and the server sends a keepalive about every 15 seconds.
49
78
  */
50
79
  stream(options?: RequestOptions): AsyncGenerator<T, void, undefined>;
51
80
  /** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
@@ -55,21 +84,63 @@ export declare class Feed<T> {
55
84
  * copied when `allExports` is called; errors, including invalid parameters, are
56
85
  * thrown from the iteration.
57
86
  */
58
- allExports({ limit, cursor, signal }?: ListExportsOptions): AsyncGenerator<Export, void, undefined>;
87
+ allExports(options?: ListExportsOptions): AsyncGenerator<Export, void, undefined>;
88
+ /** Returns the rows and bytes the feed exported per UTC day, oldest first. */
89
+ exportVolume(options?: ExportVolumeOptions): Promise<ExportVolume>;
59
90
  /** Returns the checksum, size, row count, and parquet schema of an export. */
60
- exportMeta(id: string, options?: RequestOptions): Promise<ExportMeta>;
91
+ exportMeta(id: string, options?: ExportOptions): Promise<ExportMeta>;
61
92
  /**
62
93
  * Returns a presigned URL for an export's parquet file, for tools that read parquet
63
94
  * over HTTPS directly. The URL is valid for 24 hours and grants access without an API
64
95
  * key, so treat it as a secret.
65
96
  */
66
- downloadURL(id: string, options?: RequestOptions): Promise<URL>;
97
+ downloadURL(id: string, options?: ExportOptions): Promise<URL>;
67
98
  /**
68
99
  * Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
69
100
  * to release the connection. The API key is sent only to the Synthient API, never to
70
101
  * the storage host that serves the file, and errors never include the presigned URL.
71
102
  * A non-success response from either returns an APIError.
72
103
  */
73
- download(id: string, options?: RequestOptions): Promise<ReadableStream<Uint8Array>>;
104
+ download(id: string, options?: ExportOptions): Promise<ReadableStream<Uint8Array>>;
105
+ }
106
+ /**
107
+ * A Synthient dataset, published as one export per UTC day with no stream and no hourly
108
+ * exports. Access datasets through the client, such as `client.geolocation`.
109
+ *
110
+ * Export IDs are "latest" for the newest day or a UTC day as "YYYY-MM-DD". `Export.id`
111
+ * values and the results of {@link dailyExport} can be passed as they are.
112
+ */
113
+ export declare class Dataset {
114
+ #private;
115
+ /** @internal Datasets are created by the client. */
116
+ constructor(transport: Transport, path: string);
117
+ /** Returns one page of the dataset's exports, newest first. Use `allExports` to iterate every page. */
118
+ listExports(options?: ListDatasetExportsOptions): Promise<ExportPage>;
119
+ /**
120
+ * Iterates over the dataset's exports, newest first, fetching pages lazily. `options`
121
+ * is copied when `allExports` is called; errors, including invalid parameters, are
122
+ * thrown from the iteration.
123
+ */
124
+ allExports(options?: ListDatasetExportsOptions): AsyncGenerator<Export, void, undefined>;
125
+ /** Returns the rows and bytes the dataset exported per UTC day, oldest first. */
126
+ exportVolume(options?: ExportVolumeOptions): Promise<ExportVolume>;
127
+ /**
128
+ * Returns the checksum, size, row count, and schema of an export's file in the
129
+ * requested format, and lists every file of the day.
130
+ */
131
+ exportMeta(id: string, options?: DatasetExportOptions): Promise<ExportMeta>;
132
+ /**
133
+ * Returns a presigned URL for an export's file. The URL is valid for 24 hours and
134
+ * grants access without an API key, so treat it as a secret.
135
+ */
136
+ downloadURL(id: string, options?: DatasetExportOptions): Promise<URL>;
137
+ /**
138
+ * Downloads an export's file as a byte stream, exactly as served: a "csv" file stays
139
+ * gzip-compressed and a WHOIS archive stays packed. Read it to the end or cancel it to
140
+ * release the connection. The API key is sent only to the Synthient API, never to the
141
+ * storage host that serves the file, and errors never include the presigned URL. A
142
+ * non-success response from either returns an APIError.
143
+ */
144
+ download(id: string, options?: DatasetExportOptions): Promise<ReadableStream<Uint8Array>>;
74
145
  }
75
146
  export {};
package/dist/feed.js CHANGED
@@ -36,20 +36,22 @@ export class Feed {
36
36
  #transport;
37
37
  #path;
38
38
  #reviver;
39
+ #exports;
39
40
  /** @internal Feeds are created by the client. */
40
41
  constructor(transport, path, reviver) {
41
42
  this.#transport = transport;
42
43
  this.#path = path;
43
44
  this.#reviver = reviver;
45
+ this.#exports = new Exports(transport, path, true);
44
46
  }
45
47
  /**
46
48
  * Streams the feed's real-time events. The connection opens on the first `next()`,
47
49
  * which throws any connection error, such as an APIError with status 403 for a
48
- * missing scope. The server closes each stream after up to 30 minutes, which ends the
49
- * iteration normally; any other failure throws. Breaking out of the loop closes the
50
- * connection. Events sent between connections are missed, so call `stream` again to
51
- * reconnect. The client's `timeout` bounds the gap between data, and the server sends
52
- * a keepalive about every 15 seconds.
50
+ * missing scope. The server can close a stream at any time, such as during a deploy,
51
+ * which ends the iteration normally; any other failure throws. Breaking out of the
52
+ * loop closes the connection. Events sent between connections are missed, so call
53
+ * `stream` again to reconnect. The client's `timeout` bounds the gap between data,
54
+ * and the server sends a keepalive about every 15 seconds.
53
55
  */
54
56
  async *stream(options) {
55
57
  const signal = options?.signal;
@@ -57,12 +59,119 @@ export class Feed {
57
59
  yield* ndjson(body, signal, this.#reviver);
58
60
  }
59
61
  /** Returns one page of the feed's exports, newest first. Use `allExports` to iterate every page. */
60
- async listExports(options) {
61
- const query = limitQuery(options?.limit);
62
- if (options?.cursor) {
62
+ listExports(options) {
63
+ return this.#exports.list(options);
64
+ }
65
+ /**
66
+ * Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
67
+ * copied when `allExports` is called; errors, including invalid parameters, are
68
+ * thrown from the iteration.
69
+ */
70
+ allExports(options) {
71
+ return this.#exports.all(options);
72
+ }
73
+ /** Returns the rows and bytes the feed exported per UTC day, oldest first. */
74
+ exportVolume(options) {
75
+ return this.#exports.volume(options);
76
+ }
77
+ /** Returns the checksum, size, row count, and parquet schema of an export. */
78
+ exportMeta(id, options) {
79
+ return this.#exports.meta(id, options);
80
+ }
81
+ /**
82
+ * Returns a presigned URL for an export's parquet file, for tools that read parquet
83
+ * over HTTPS directly. The URL is valid for 24 hours and grants access without an API
84
+ * key, so treat it as a secret.
85
+ */
86
+ downloadURL(id, options) {
87
+ return this.#exports.downloadURL(id, options);
88
+ }
89
+ /**
90
+ * Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
91
+ * to release the connection. The API key is sent only to the Synthient API, never to
92
+ * the storage host that serves the file, and errors never include the presigned URL.
93
+ * A non-success response from either returns an APIError.
94
+ */
95
+ download(id, options) {
96
+ return this.#exports.download(id, options);
97
+ }
98
+ }
99
+ /**
100
+ * A Synthient dataset, published as one export per UTC day with no stream and no hourly
101
+ * exports. Access datasets through the client, such as `client.geolocation`.
102
+ *
103
+ * Export IDs are "latest" for the newest day or a UTC day as "YYYY-MM-DD". `Export.id`
104
+ * values and the results of {@link dailyExport} can be passed as they are.
105
+ */
106
+ export class Dataset {
107
+ #exports;
108
+ /** @internal Datasets are created by the client. */
109
+ constructor(transport, path) {
110
+ this.#exports = new Exports(transport, path, false);
111
+ }
112
+ /** Returns one page of the dataset's exports, newest first. Use `allExports` to iterate every page. */
113
+ listExports(options) {
114
+ return this.#exports.list(options);
115
+ }
116
+ /**
117
+ * Iterates over the dataset's exports, newest first, fetching pages lazily. `options`
118
+ * is copied when `allExports` is called; errors, including invalid parameters, are
119
+ * thrown from the iteration.
120
+ */
121
+ allExports(options) {
122
+ return this.#exports.all(options);
123
+ }
124
+ /** Returns the rows and bytes the dataset exported per UTC day, oldest first. */
125
+ exportVolume(options) {
126
+ return this.#exports.volume(options);
127
+ }
128
+ /**
129
+ * Returns the checksum, size, row count, and schema of an export's file in the
130
+ * requested format, and lists every file of the day.
131
+ */
132
+ exportMeta(id, options) {
133
+ return this.#exports.meta(id, options);
134
+ }
135
+ /**
136
+ * Returns a presigned URL for an export's file. The URL is valid for 24 hours and
137
+ * grants access without an API key, so treat it as a secret.
138
+ */
139
+ downloadURL(id, options) {
140
+ return this.#exports.downloadURL(id, options);
141
+ }
142
+ /**
143
+ * Downloads an export's file as a byte stream, exactly as served: a "csv" file stays
144
+ * gzip-compressed and a WHOIS archive stays packed. Read it to the end or cancel it to
145
+ * release the connection. The API key is sent only to the Synthient API, never to the
146
+ * storage host that serves the file, and errors never include the presigned URL. A
147
+ * non-success response from either returns an APIError.
148
+ */
149
+ download(id, options) {
150
+ return this.#exports.download(id, options);
151
+ }
152
+ }
153
+ /** The export routes that feeds and datasets share under their path. */
154
+ class Exports {
155
+ #transport;
156
+ #path;
157
+ #hourly;
158
+ constructor(transport, path, hourly) {
159
+ this.#transport = transport;
160
+ this.#path = path;
161
+ this.#hourly = hourly;
162
+ }
163
+ async list(options = {}) {
164
+ const query = limitQuery(options.limit);
165
+ if (options.cursor) {
63
166
  query.cursor = options.cursor;
64
167
  }
65
- const page = await this.#transport.json({ path: `${this.#path}/export`, query, policy: "transient" }, options?.signal);
168
+ if (options.day !== undefined) {
169
+ query.day = options.day;
170
+ }
171
+ if (options.variant !== undefined) {
172
+ query.variant = options.variant;
173
+ }
174
+ const page = await this.#transport.json({ path: `${this.#path}/export`, query, policy: "transient" }, options.signal);
66
175
  const cursor = page.next_cursor;
67
176
  if (!Array.isArray(page.feeds)) {
68
177
  throw new SynthientError("export page has no feeds array");
@@ -72,19 +181,14 @@ export class Feed {
72
181
  }
73
182
  return page;
74
183
  }
75
- /**
76
- * Iterates over the feed's exports, newest first, fetching pages lazily. `options` is
77
- * copied when `allExports` is called; errors, including invalid parameters, are
78
- * thrown from the iteration.
79
- */
80
- async *allExports({ limit, cursor, signal } = {}) {
184
+ async *all({ cursor, ...options } = {}) {
81
185
  const seen = new Set([cursor ?? ""]);
82
186
  for (;;) {
83
- const page = await this.listExports({ limit, cursor, signal });
84
- signal?.throwIfAborted();
187
+ const page = await this.list({ ...options, cursor });
188
+ options.signal?.throwIfAborted();
85
189
  for (const item of page.feeds) {
86
190
  yield item;
87
- signal?.throwIfAborted();
191
+ options.signal?.throwIfAborted();
88
192
  }
89
193
  if (!page.next_cursor) {
90
194
  return;
@@ -96,56 +200,73 @@ export class Feed {
96
200
  cursor = page.next_cursor;
97
201
  }
98
202
  }
99
- /** Returns the checksum, size, row count, and parquet schema of an export. */
100
- async exportMeta(id, options) {
101
- return this.#transport.json({ path: `${this.#path}/export/${exportPath(id)}/meta`, policy: "transient" }, options?.signal);
203
+ async volume(options = {}) {
204
+ const query = {};
205
+ if (options.days !== undefined) {
206
+ query.days = String(options.days);
207
+ }
208
+ return this.#transport.json({ path: `${this.#path}/export/daily`, query, policy: "transient" }, options.signal);
102
209
  }
103
- /**
104
- * Returns a presigned URL for an export's parquet file, for tools that read parquet
105
- * over HTTPS directly. The URL is valid for 24 hours and grants access without an API
106
- * key, so treat it as a secret.
107
- */
108
- async downloadURL(id, options) {
210
+ async meta(id, options = {}) {
211
+ const meta = await this.#transport.json({
212
+ path: `${this.#path}/export/${this.#exportPath(id)}/meta`,
213
+ query: fileQuery(options),
214
+ policy: "transient",
215
+ }, options.signal);
216
+ // The WHOIS meta once sent `"fields": null` for its archive's empty schema.
217
+ for (const { schema } of [meta, ...(meta.entities ?? [])]) {
218
+ if (schema) {
219
+ schema.fields = Array.isArray(schema.fields) ? schema.fields : [];
220
+ }
221
+ }
222
+ return meta;
223
+ }
224
+ async downloadURL(id, options = {}) {
109
225
  const request = {
110
- path: `${this.#path}/export/${exportPath(id)}`,
226
+ path: `${this.#path}/export/${this.#exportPath(id)}`,
227
+ query: fileQuery(options),
111
228
  policy: "transient",
112
229
  };
113
- const [response, attempt] = await this.#transport.send(request, options?.signal);
230
+ const [response, attempt] = await this.#transport.send(request, options.signal);
114
231
  if (response.status >= 400) {
115
232
  throw await this.#transport.error(response, attempt);
116
233
  }
117
234
  await drain(response, attempt);
118
- if (response.status < 300) {
235
+ if (response.status < 300 || response.status === 304) {
119
236
  throw new SynthientError(`expected a redirect to the export, got status ${response.status}`);
120
237
  }
121
238
  return redirectTarget(new URL(this.#transport.url(request)), response);
122
239
  }
123
- /**
124
- * Downloads an export's parquet file as a byte stream. Read it to the end or cancel it
125
- * to release the connection. The API key is sent only to the Synthient API, never to
126
- * the storage host that serves the file, and errors never include the presigned URL.
127
- * A non-success response from either returns an APIError.
128
- */
129
- async download(id, options) {
240
+ async download(id, options = {}) {
130
241
  const url = await this.downloadURL(id, options);
131
- return this.#transport.download(url, options?.signal);
242
+ return this.#transport.download(url, options.signal);
132
243
  }
133
- }
134
- /** Converts an export ID to its URL path, with the hour unpadded. */
135
- function exportPath(id) {
136
- if (id === "latest") {
137
- return id;
138
- }
139
- const match = EXPORT_ID.exec(id);
140
- if (match) {
141
- const [, year, month, day, hour] = match;
142
- // An invalid day or an hour past 23 rolls the date over, so it no longer matches.
143
- const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour ?? 0)));
144
- if (dailyExport(date) === id.slice(0, 10)) {
145
- return hour === undefined ? dailyExport(date) : hourlyExport(date);
244
+ /** Converts an export ID to its URL path, with the hour unpadded. */
245
+ #exportPath(id) {
246
+ if (id === "latest") {
247
+ return id;
146
248
  }
249
+ const match = EXPORT_ID.exec(id);
250
+ if (match && (this.#hourly || match[4] === undefined)) {
251
+ const [, year, month, day, hour] = match;
252
+ // An invalid day or an hour past 23 rolls the date over, so it no longer matches.
253
+ const date = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day), Number(hour ?? 0)));
254
+ if (dailyExport(date) === id.slice(0, 10)) {
255
+ return hour === undefined ? dailyExport(date) : hourlyExport(date);
256
+ }
257
+ }
258
+ throw new TypeError("invalid export ID");
259
+ }
260
+ }
261
+ function fileQuery({ variant, format }) {
262
+ const query = {};
263
+ if (variant !== undefined) {
264
+ query.variant = variant;
265
+ }
266
+ if (format !== undefined) {
267
+ query.format = format;
147
268
  }
148
- throw new TypeError("invalid export ID");
269
+ return query;
149
270
  }
150
271
  /** Blank lines are keepalives. A final line without a newline is still an event. */
151
272
  async function* ndjson(body, signal, reviver) {
@@ -2,7 +2,7 @@ import { type FileDescriptorSet } from "@bufbuild/protobuf/wkt";
2
2
  import { SynthientError } from "./errors.ts";
3
3
  export declare const DEFAULT_ENDPOINT = "grpc.synthient.com:443";
4
4
  export interface FetchSchemaOptions {
5
- /** Sent as x-api-key metadata when not empty. Defaults to the SYNTHIENT_API_KEY environment variable. */
5
+ /** Sent as x-api-key metadata, which server reflection requires. Defaults to the SYNTHIENT_API_KEY environment variable. */
6
6
  apiKey?: string | undefined;
7
7
  /**
8
8
  * The server as host:port, a bare host (port 443 is assumed), or a grpc:// or
package/dist/index.d.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  /**
2
- * The official Node.js client for the Synthient API: IP and domain intelligence,
3
- * account details, and real-time and bulk data feeds.
2
+ * The official Node.js client for the Synthient API: IP intelligence, account details,
3
+ * and real-time and bulk data feeds.
4
4
  *
5
5
  * @example
6
6
  * ```ts
@@ -16,7 +16,7 @@
16
16
  export { DEFAULT_BASE_URL, Synthient, type ClientOptions, type LookupDomainOptions, } from "./client.ts";
17
17
  export { APIConnectionError, APIError, APITimeoutError, SynthientError, type APIErrorInit, } from "./errors.ts";
18
18
  export type * from "./events.ts";
19
- export { Feed, dailyExport, hourlyExport, type ListExportsOptions } from "./feed.ts";
19
+ export { Dataset, Feed, dailyExport, hourlyExport, type DatasetExportOptions, type ExportOptions, type ExportVolumeOptions, type ListDatasetExportsOptions, type ListExportsOptions, } from "./feed.ts";
20
20
  export type { Fetch, RequestOptions, ResponseMetadata } from "./transport.ts";
21
21
  export { toDate } from "./types.ts";
22
22
  export type * from "./types.ts";
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  /**
2
- * The official Node.js client for the Synthient API: IP and domain intelligence,
3
- * account details, and real-time and bulk data feeds.
2
+ * The official Node.js client for the Synthient API: IP intelligence, account details,
3
+ * and real-time and bulk data feeds.
4
4
  *
5
5
  * @example
6
6
  * ```ts
@@ -15,6 +15,6 @@
15
15
  */
16
16
  export { DEFAULT_BASE_URL, Synthient, } from "./client.js";
17
17
  export { APIConnectionError, APIError, APITimeoutError, SynthientError, } from "./errors.js";
18
- export { Feed, dailyExport, hourlyExport } from "./feed.js";
18
+ export { Dataset, Feed, dailyExport, hourlyExport, } from "./feed.js";
19
19
  export { toDate } from "./types.js";
20
20
  export { VERSION } from "./version.js";
package/dist/types.d.ts CHANGED
@@ -8,15 +8,28 @@ export type UnixTime = number;
8
8
  export type NetworkType = "MOBILE" | "SATELLITE" | "IN_FLIGHT_WIFI" | "RESIDENTIAL" | "CORPORATE" | "ACADEMIC" | "DATACENTER" | "GOVERNMENT" | (string & {});
9
9
  /** New values may be added. */
10
10
  export type DeviceOS = "Android" | "Chrome OS" | "FreeBSD" | "iOS" | "Linux" | "Mac OS X" | "Windows" | (string & {});
11
- /** New values may be added. */
12
- export type Scope = "BASIC" | "PROXY_FEEDS" | "PROXY_FIREHOSE" | "ANONYMIZERS_FEED" | "ANONYMIZERS_STREAM" | "JA3_FEED" | "JA3_STREAM" | "TORRENTS_FEED" | "TORRENTS_STREAM" | "HONEYPOT_HTTP_FEED" | "HONEYPOT_HTTP_STREAM" | "HONEYPOT_HTTPS_FEED" | "HONEYPOT_HTTPS_STREAM" | "HONEYPOT_ADB_FEED" | "HONEYPOT_ADB_STREAM" | "HONEYPOT_DNS_FEED" | "HONEYPOT_DNS_STREAM" | "JA4_FEED" | "JA4_STREAM" | "JA4T_FEED" | "JA4T_STREAM" | "ADMIN_PROXIES" | (string & {});
11
+ /**
12
+ * New values may be added. WHOIS_STREAM, NETWORK_IOCS_FEED, NETWORK_IOCS_STREAM,
13
+ * FILE_IOCS_FEED, and FILE_IOCS_STREAM are reserved for upcoming feeds, and HELIOS is a
14
+ * dashboard permission that authorizes no API operation. ADMIN_PROXIES is deprecated:
15
+ * it never authorized an API operation, the API no longer lists it, and it will be
16
+ * removed in the next major version.
17
+ */
18
+ export type Scope = "BASIC" | "PROXY_FEEDS" | "PROXY_FIREHOSE" | "ANONYMIZERS_FEED" | "ANONYMIZERS_STREAM" | "JA3_FEED" | "JA3_STREAM" | "TORRENTS_FEED" | "TORRENTS_STREAM" | "HONEYPOT_HTTP_FEED" | "HONEYPOT_HTTP_STREAM" | "HONEYPOT_HTTPS_FEED" | "HONEYPOT_HTTPS_STREAM" | "HONEYPOT_ADB_FEED" | "HONEYPOT_ADB_STREAM" | "HONEYPOT_DNS_FEED" | "HONEYPOT_DNS_STREAM" | "JA4_FEED" | "JA4_STREAM" | "JA4T_FEED" | "JA4T_STREAM" | "HELIOS" | "WHOIS_FEED" | "WHOIS_STREAM" | "GEOLOCATION_FEED" | "NETWORK_IOCS_FEED" | "NETWORK_IOCS_STREAM" | "FILE_IOCS_FEED" | "FILE_IOCS_STREAM" | "ADMIN_PROXIES" | (string & {});
13
19
  /** The API always sends one of these sentences. */
14
20
  export type ActivityCadence = "No activity in the last 30 days." | "Activity is evenly distributed across the day." | "Activity follows a weekday business-hours pattern." | "Activity follows a weekend pattern." | "Activity is burst-driven with significant peaks.";
15
- /** New values may be added. */
21
+ /**
22
+ * New values may be added. The datasets also send types such as "double", "date",
23
+ * "timestamp", and "list<string>".
24
+ */
16
25
  export type ParquetType = "string" | "int64" | "uint32" | "uint64" | "bool" | "bytes" | (string & {});
17
26
  /** Converts a {@link UnixTime} to a Date, reading values of 1e12 or more as milliseconds. */
18
27
  export declare function toDate(timestamp: UnixTime): Date;
19
- /** `network`, `location`, and `intelligence` are absent when the API has no data for them. */
28
+ /**
29
+ * `network` is absent when the address lies in no known network range or the API's
30
+ * enrichment lookup failed, and `location` is absent along with it. `intelligence` is
31
+ * absent when the API has no data for it.
32
+ */
20
33
  export interface IPInfo {
21
34
  ip: string;
22
35
  network?: Network;
@@ -77,7 +90,12 @@ export interface Device {
77
90
  version: string;
78
91
  last_seen: UnixTime;
79
92
  }
80
- /** The honeypot traffic Synthient has observed for a domain. */
93
+ /**
94
+ * The honeypot traffic Synthient has observed for a domain.
95
+ *
96
+ * @deprecated Domain lookup was removed from the Synthient API and will stop working when
97
+ * the endpoint is turned off. It will be removed in the next major version.
98
+ */
81
99
  export interface DomainInfo {
82
100
  domain: string;
83
101
  /**
@@ -189,6 +207,23 @@ export interface AccountAPIKey {
189
207
  /** When the key was created, as an RFC 3339 string. */
190
208
  created_at: string;
191
209
  }
210
+ /** Whether an export covers a UTC day or a UTC hour. */
211
+ export type ExportKind = "hourly" | "daily";
212
+ /**
213
+ * The format of an export file. Feeds have only "parquet". The geolocation dataset adds
214
+ * "csv", which is gzip-compressed, and "mmdb", a MaxMind DB. The WHOIS dataset is one
215
+ * "tar.gz" archive of parquet files.
216
+ */
217
+ export type ExportFormat = "parquet" | "csv" | "mmdb" | "tar.gz";
218
+ /**
219
+ * "full" is every sighting. "lite" is the snapshot deduplicated to the latest row in its
220
+ * hour or day, with the same columns: one row per address and provider for proxies, per
221
+ * address, fingerprint, and provider for JA4T, and per range and provider for
222
+ * anonymizers.
223
+ */
224
+ export type ExportVariant = "full" | "lite";
225
+ /** A parquet file inside a WHOIS archive. */
226
+ export type WhoisEntity = "asns" | "networks" | "organisations" | "contacts" | "maintainers" | "transfers" | "delegations";
192
227
  /** One page of a feed's exports, newest first. */
193
228
  export interface ExportPage {
194
229
  /** The feed's stream identifier, such as "proxies". */
@@ -198,7 +233,7 @@ export interface ExportPage {
198
233
  /** Pass it as `cursor` to fetch the next page. It is absent, null, or "" on the last page. */
199
234
  next_cursor?: string | null;
200
235
  }
201
- /** A parquet export of a feed. */
236
+ /** An export of a feed or dataset. */
202
237
  export interface Export {
203
238
  /**
204
239
  * Identifies the export: "YYYY-MM-DD" for a daily export, "YYYY-MM-DD/H" with an
@@ -206,37 +241,95 @@ export interface Export {
206
241
  * first page. Pass it to `download`.
207
242
  */
208
243
  id: string;
209
- /** Whether the export covers a UTC day or a UTC hour. */
210
- kind: "hourly" | "daily";
244
+ kind: ExportKind;
211
245
  /** The UTC day of the export as "YYYY-MM-DD". */
212
246
  date: string;
213
247
  /** The UTC hour of an hourly export, absent or null for a daily export. */
214
248
  hour?: number | null;
249
+ /** The size in bytes of the first file in `formats`. */
215
250
  size_bytes: number;
251
+ /** The number of rows. For WHOIS, the sum of its entities' rows. */
216
252
  row_count: number;
217
- /** The hex-encoded SHA-256 of the parquet file. */
253
+ /** The hex-encoded SHA-256 of the first file in `formats`. */
218
254
  checksum: string;
219
255
  created_at: UnixTime;
220
256
  /** The API path of the export's download, including the version prefix. */
221
257
  download_path: string;
258
+ /** Every file of the export, its own format first. */
259
+ formats: ExportFile[];
260
+ /** The parquet files inside a WHOIS archive, absent for other exports. */
261
+ entities?: ExportEntity[];
262
+ /**
263
+ * "lite" in a lite listing, where the sizes, checksum, and `download_path` describe the
264
+ * lite file, and absent otherwise. `id` is the same for both variants, so pass the
265
+ * variant again to download the lite file.
266
+ */
267
+ variant?: ExportVariant;
222
268
  }
223
269
  export interface ExportMeta {
224
270
  id: string;
225
271
  stream: string;
226
- /** Whether the export covers a UTC day or a UTC hour. */
227
- kind: "hourly" | "daily";
228
- format: "parquet";
272
+ kind: ExportKind;
273
+ /** The format of the file that `checksum`, `size`, and `schema` describe. */
274
+ format: ExportFormat;
229
275
  /** The start of the export: midnight UTC for a daily export, the hour for an hourly one. */
230
276
  date: UnixTime;
231
277
  /** The UTC hour of an hourly export, absent or null for a daily export. */
232
278
  hour?: number | null;
233
279
  rows: number;
234
- /** The size of the parquet file in bytes. */
280
+ /** The size of the file in bytes. */
235
281
  size: number;
236
- /** The hex-encoded SHA-256 of the parquet file. */
282
+ /** The hex-encoded SHA-256 of the file. */
237
283
  checksum: string;
238
284
  created_at: UnixTime;
285
+ /** The file's columns. A WHOIS archive has none; its parquet files list theirs in `entities`. */
239
286
  schema: ParquetSchema;
287
+ /** Every file of the export, its own format first. */
288
+ formats: ExportFile[];
289
+ /** The parquet files inside a WHOIS archive, each with its columns, absent for other exports. */
290
+ entities?: ExportEntity[];
291
+ /**
292
+ * "lite" for a lite export, where the checksum, size, rows, schema, and formats describe
293
+ * the lite file, and absent otherwise.
294
+ */
295
+ variant?: ExportVariant;
296
+ }
297
+ /** One file of an export. */
298
+ export interface ExportFile {
299
+ format: ExportFormat;
300
+ size_bytes: number;
301
+ /** The hex-encoded SHA-256 of the file as downloaded, so of the gzip file for "csv". */
302
+ checksum: string;
303
+ }
304
+ /** A parquet file inside a WHOIS archive. */
305
+ export interface ExportEntity {
306
+ entity: WhoisEntity;
307
+ row_count: number;
308
+ size_bytes: number;
309
+ /** The hex-encoded SHA-256 of the parquet file. */
310
+ checksum: string;
311
+ /** The parquet file's columns, present only in `exportMeta` results. */
312
+ schema?: ParquetSchema;
313
+ }
314
+ /** The rows and bytes a feed or dataset exported per UTC day. */
315
+ export interface ExportVolume {
316
+ /** The feed's stream identifier, such as "proxies". */
317
+ stream: string;
318
+ /** Oldest first. Today is included while in progress, and days without exports are left out. */
319
+ days: ExportVolumeDay[];
320
+ }
321
+ export interface ExportVolumeDay {
322
+ /** The UTC day as "YYYY-MM-DD". */
323
+ date: string;
324
+ /**
325
+ * "daily" when the day's rollup is counted. "hourly" when its hourly exports are summed
326
+ * instead, as for today or a day whose rollup is still pending.
327
+ */
328
+ kind: ExportKind;
329
+ /** The number of files counted: 1 for a rollup, otherwise the hourly exports so far, of 24. */
330
+ snapshots: number;
331
+ row_count: number;
332
+ size_bytes: number;
240
333
  }
241
334
  export interface ParquetSchema {
242
335
  /** The columns in file order. */
package/dist/version.d.ts CHANGED
@@ -1,2 +1,2 @@
1
1
  /** The client sends it in the User-Agent header. */
2
- export declare const VERSION = "2.0.0";
2
+ export declare const VERSION = "2.1.0";
package/dist/version.js CHANGED
@@ -1,2 +1,2 @@
1
1
  /** The client sends it in the User-Agent header. */
2
- export const VERSION = "2.0.0";
2
+ export const VERSION = "2.1.0";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@synthient/synthient",
3
- "version": "2.0.0",
3
+ "version": "2.1.0",
4
4
  "description": "The official Node.js client for the Synthient API.",
5
5
  "keywords": [
6
6
  "synthient",