@thirdweb-dev/service-utils 0.8.5-nightly-821caa63d78a2f985925d6c1be24143f46f435f4-20250205033244 → 0.8.5-nightly-ee6bd4f8982d2b0db823c8cb472aeb9d6f9408fb-20250205060502
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/node/index.js +1 -0
- package/dist/cjs/node/index.js.map +1 -1
- package/dist/cjs/node/kafka.js +120 -0
- package/dist/cjs/node/kafka.js.map +1 -0
- package/dist/cjs/node/usageV2.js +23 -103
- package/dist/cjs/node/usageV2.js.map +1 -1
- package/dist/esm/node/index.js +1 -0
- package/dist/esm/node/index.js.map +1 -1
- package/dist/esm/node/kafka.js +116 -0
- package/dist/esm/node/kafka.js.map +1 -0
- package/dist/esm/node/usageV2.js +23 -103
- package/dist/esm/node/usageV2.js.map +1 -1
- package/dist/types/node/index.d.ts +1 -0
- package/dist/types/node/index.d.ts.map +1 -1
- package/dist/types/node/kafka.d.ts +55 -0
- package/dist/types/node/kafka.d.ts.map +1 -0
- package/dist/types/node/usageV2.d.ts +5 -18
- package/dist/types/node/usageV2.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/cjs/node/index.js
CHANGED
|
@@ -7,6 +7,7 @@ exports.logHttpRequest = logHttpRequest;
|
|
|
7
7
|
const tslib_1 = require("tslib");
|
|
8
8
|
const node_crypto_1 = require("node:crypto");
|
|
9
9
|
const index_js_1 = require("../core/authorize/index.js");
|
|
10
|
+
tslib_1.__exportStar(require("./kafka.js"), exports);
|
|
10
11
|
tslib_1.__exportStar(require("./usageV2.js"), exports);
|
|
11
12
|
tslib_1.__exportStar(require("../core/usage.js"), exports);
|
|
12
13
|
tslib_1.__exportStar(require("../core/usageV2.js"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":";;AA4BA,sCA0BC;AAmBD,4DAkGC;AAED,sCAEC;AAED,wCAsCC;;AAvND,6CAAyC;AAOzC,yDAAuD;AAQvD,qDAA2B;AAC3B,uDAA6B;AAC7B,2DAAiC;AACjC,6DAAmC;AACnC,qEAA2C;AAC3C,8DAAoC;AAQ7B,KAAK,UAAU,aAAa,CACjC,SAAoB,EACpB,aAAgC,EAChC,YAA2B;IAE3B,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,QAAQ,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;YACvD,OAAO;gBACL,UAAU,EAAE,KAAK;gBACjB,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,iDAAiD;gBAC/D,SAAS,EAAE,cAAc;aAC1B,CAAC;QACJ,CAAC;QACD,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,MAAM,EAAE,GAAG;YACX,YAAY,EAAE,uBAAuB;YACrC,SAAS,EAAE,uBAAuB;SACnC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,IAAA,oBAAS,EAAC,QAAQ,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CACpB,OAAsC;IAEtC,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAChB,OAAsC,EACtC,UAAkB;IAElB,OAAO,aAAa,CAAC,OAAO,CAAC;QAC3B,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACpC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;QACjC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,SAAgB,wBAAwB,CACtC,SAAoB;IAEpB,IAAI,UAAe,CAAC;IAEpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,GAAG,CAClB,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EACvB,UAAU,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC;IACtC,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACrD,+CAA+C;IAC/C,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE1C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,0CAA0C;QAC1C,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,4CAA4C;IAC5C,IAAI,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEjD,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,WAAW,GAAG,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACvD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,kBAAkB,GAAG,SAAS,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACtE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,kBAAkB,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC1C,iEAAiE;IACjE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IACD,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;QAChC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,SAAS,EAAE,CAAC;QACd,sBAAsB;QACtB,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,GAAG,GAAkB,IAAI,CAAC;IAC9B,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,gDAAgD;IAChD,MAAM,mBAAmB,GAAG,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,mBAAmB,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAChD,GAAG,GAAG,KAAK,CAAC;YACZ,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;YAClE,sFAAsF;YACtF,IAAI,gBAAgB,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBAC/C,aAAa,GAAG,gBAAgB,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,GAAG;QACH,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1C,aAAa;QACb,SAAS;QACT,QAAQ;QACR,WAAW;QACX,kBAAkB;QAClB,MAAM;QACN,QAAQ;QACR,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,aAAa;KACd,CAAC;AACJ,CAAC;AAED,SAAgB,aAAa,CAAC,SAAiB;IAC7C,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,SAAgB,cAAc,CAAC,EAC7B,QAAQ,EACR,GAAG,EACH,GAAG,EACH,QAAQ,EACR,aAAa,EACb,SAAS,GAQV;IACC,IAAI,CAAC;QACH,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,GAAG;YACjB,YAAY,EAAE,CAAC,CAAC,iBAAiB,CAAC,SAAS;YAC3C,WAAW,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ;YACzC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG;YAC/B,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,GAAG,CAAC,UAAU;YACtB,aAAa;YACb,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;YACzC,UAAU,EAAE,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC;YAC/C,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC;YAC9C,EAAE,EAAE,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC;YAClC,SAAS;SACV,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.KafkaProducer = void 0;
|
|
4
|
+
const node_tls_1 = require("node:tls");
|
|
5
|
+
const kafkajs_1 = require("kafkajs");
|
|
6
|
+
const lz4js_1 = require("lz4js");
|
|
7
|
+
// CompressionCodecs is not exported properly in kafkajs. Source: https://github.com/tulios/kafkajs/issues/1391
|
|
8
|
+
const kafkajs_2 = require("kafkajs");
|
|
9
|
+
const { CompressionCodecs } = kafkajs_2.default;
|
|
10
|
+
/**
|
|
11
|
+
* Creates a KafkaProducer which opens a persistent TCP connection.
|
|
12
|
+
* This class is thread-safe so your service should re-use one instance.
|
|
13
|
+
*
|
|
14
|
+
* Example:
|
|
15
|
+
* ```ts
|
|
16
|
+
* kafka = new KafkaProducer(...)
|
|
17
|
+
* await kafka.send(topic, events)
|
|
18
|
+
* // Non-blocking:
|
|
19
|
+
* // void kafka.send(topic, events).catch((e) => console.error(e))
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
class KafkaProducer {
|
|
23
|
+
constructor(config) {
|
|
24
|
+
Object.defineProperty(this, "kafka", {
|
|
25
|
+
enumerable: true,
|
|
26
|
+
configurable: true,
|
|
27
|
+
writable: true,
|
|
28
|
+
value: void 0
|
|
29
|
+
});
|
|
30
|
+
Object.defineProperty(this, "producer", {
|
|
31
|
+
enumerable: true,
|
|
32
|
+
configurable: true,
|
|
33
|
+
writable: true,
|
|
34
|
+
value: null
|
|
35
|
+
});
|
|
36
|
+
Object.defineProperty(this, "compression", {
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true,
|
|
40
|
+
value: void 0
|
|
41
|
+
});
|
|
42
|
+
const { producerName, environment, shouldCompress = true, username, password, } = config;
|
|
43
|
+
this.kafka = new kafkajs_1.Kafka({
|
|
44
|
+
clientId: `${producerName}-${environment}`,
|
|
45
|
+
brokers: environment === "production"
|
|
46
|
+
? ["warpstream.thirdweb.xyz:9092"]
|
|
47
|
+
: ["warpstream-dev.thirdweb.xyz:9092"],
|
|
48
|
+
ssl: {
|
|
49
|
+
checkServerIdentity(hostname, cert) {
|
|
50
|
+
return (0, node_tls_1.checkServerIdentity)(hostname.toLowerCase(), cert);
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
sasl: {
|
|
54
|
+
mechanism: "plain",
|
|
55
|
+
username,
|
|
56
|
+
password,
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
if (shouldCompress) {
|
|
60
|
+
this.compression = kafkajs_1.CompressionTypes.LZ4;
|
|
61
|
+
CompressionCodecs[kafkajs_1.CompressionTypes.LZ4] = () => ({
|
|
62
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
63
|
+
compress: (encoder) => {
|
|
64
|
+
const compressed = (0, lz4js_1.compress)(encoder.buffer);
|
|
65
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
66
|
+
return Buffer.from(compressed);
|
|
67
|
+
},
|
|
68
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
69
|
+
decompress: (buffer) => {
|
|
70
|
+
const decompressed = (0, lz4js_1.decompress)(buffer);
|
|
71
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
72
|
+
return Buffer.from(decompressed);
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
this.compression = kafkajs_1.CompressionTypes.None;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Send messages to a Kafka topic.
|
|
82
|
+
* This method may throw. To call this non-blocking:
|
|
83
|
+
* @param topic
|
|
84
|
+
* @param messages
|
|
85
|
+
* @param configOverrides
|
|
86
|
+
*/
|
|
87
|
+
async send(topic, messages,
|
|
88
|
+
/**
|
|
89
|
+
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
90
|
+
*/
|
|
91
|
+
options) {
|
|
92
|
+
if (!this.producer) {
|
|
93
|
+
this.producer = this.kafka.producer({
|
|
94
|
+
allowAutoTopicCreation: options?.allowAutoTopicCreation ?? false,
|
|
95
|
+
});
|
|
96
|
+
await this.producer.connect();
|
|
97
|
+
}
|
|
98
|
+
await this.producer.send({
|
|
99
|
+
topic,
|
|
100
|
+
messages: messages.map((m) => ({
|
|
101
|
+
value: JSON.stringify(m),
|
|
102
|
+
})),
|
|
103
|
+
compression: this.compression,
|
|
104
|
+
acks: options?.acks ?? -1, // Default: All brokers must acknowledge
|
|
105
|
+
timeout: options?.timeout ?? 10_000, // Default: 10 seconds
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Disconnects KafkaProducer.
|
|
110
|
+
* Useful when shutting down the service to flush in-flight events.
|
|
111
|
+
*/
|
|
112
|
+
async disconnect() {
|
|
113
|
+
if (this.producer) {
|
|
114
|
+
await this.producer.disconnect();
|
|
115
|
+
this.producer = null;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
exports.KafkaProducer = KafkaProducer;
|
|
120
|
+
//# sourceMappingURL=kafka.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/node/kafka.ts"],"names":[],"mappings":";;;AAAA,uCAA+C;AAC/C,qCAAiE;AACjE,iCAA6C;AAE7C,+GAA+G;AAC/G,qCAA8B;AAC9B,MAAM,EAAE,iBAAiB,EAAE,GAAG,iBAAO,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,MAAa,aAAa;IAKxB,YAAY,MAgBX;QApBO;;;;;WAAa;QACb;;;;mBAA4B,IAAI;WAAC;QACjC;;;;;WAA8B;QAmBpC,MAAM,EACJ,YAAY,EACZ,WAAW,EACX,cAAc,GAAG,IAAI,EACrB,QAAQ,EACR,QAAQ,GACT,GAAG,MAAM,CAAC;QAEX,IAAI,CAAC,KAAK,GAAG,IAAI,eAAK,CAAC;YACrB,QAAQ,EAAE,GAAG,YAAY,IAAI,WAAW,EAAE;YAC1C,OAAO,EACL,WAAW,KAAK,YAAY;gBAC1B,CAAC,CAAC,CAAC,8BAA8B,CAAC;gBAClC,CAAC,CAAC,CAAC,kCAAkC,CAAC;YAC1C,GAAG,EAAE;gBACH,mBAAmB,CAAC,QAAQ,EAAE,IAAI;oBAChC,OAAO,IAAA,8BAAmB,EAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC3D,CAAC;aACF;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,OAAO;gBAClB,QAAQ;gBACR,QAAQ;aACT;SACF,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,CAAC,WAAW,GAAG,0BAAgB,CAAC,GAAG,CAAC;YAExC,iBAAiB,CAAC,0BAAgB,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;gBAC/C,wEAAwE;gBACxE,QAAQ,EAAE,CAAC,OAA2B,EAAE,EAAE;oBACxC,MAAM,UAAU,GAAG,IAAA,gBAAQ,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC5C,wEAAwE;oBACxE,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACjC,CAAC;gBACD,wEAAwE;gBACxE,UAAU,EAAE,CAAC,MAAc,EAAE,EAAE;oBAC7B,MAAM,YAAY,GAAG,IAAA,kBAAU,EAAC,MAAM,CAAC,CAAC;oBACxC,wEAAwE;oBACxE,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACnC,CAAC;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,0BAAgB,CAAC,IAAI,CAAC;QAC3C,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CACR,KAAa,EACb,QAAmC;IACnC;;OAEG;IACH,OAIC;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClC,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,KAAK;aACjE,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACvB,KAAK;YACL,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;aACzB,CAAC,CAAC;YACH,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,wCAAwC;YACnE,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM,EAAE,sBAAsB;SAC5D,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;CACF;AArHD,sCAqHC"}
|
package/dist/cjs/node/usageV2.js
CHANGED
|
@@ -2,13 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.UsageV2Producer = void 0;
|
|
4
4
|
const node_crypto_1 = require("node:crypto");
|
|
5
|
-
const node_tls_1 = require("node:tls");
|
|
6
|
-
const kafkajs_1 = require("kafkajs");
|
|
7
|
-
const lz4js_1 = require("lz4js");
|
|
8
5
|
const usageV2_js_1 = require("../core/usageV2.js");
|
|
9
|
-
|
|
10
|
-
const kafkajs_2 = require("kafkajs");
|
|
11
|
-
const { CompressionCodecs } = kafkajs_2.default;
|
|
6
|
+
const kafka_js_1 = require("./kafka.js");
|
|
12
7
|
/**
|
|
13
8
|
* Creates a UsageV2Producer which opens a persistent TCP connection.
|
|
14
9
|
* This class is thread-safe so your service should re-use one instance.
|
|
@@ -16,136 +11,61 @@ const { CompressionCodecs } = kafkajs_2.default;
|
|
|
16
11
|
* Example:
|
|
17
12
|
* ```ts
|
|
18
13
|
* usageV2 = new UsageV2Producer(..)
|
|
19
|
-
* await usageV2.init()
|
|
20
14
|
* await usageV2.sendEvents(events)
|
|
21
15
|
* // Non-blocking:
|
|
22
|
-
* // void usageV2.sendEvents(events).catch(console.error)
|
|
16
|
+
* // void usageV2.sendEvents(events).catch((e) => console.error(e))
|
|
23
17
|
* ```
|
|
24
18
|
*/
|
|
25
19
|
class UsageV2Producer {
|
|
26
20
|
constructor(config) {
|
|
27
|
-
Object.defineProperty(this, "
|
|
21
|
+
Object.defineProperty(this, "kafkaProducer", {
|
|
28
22
|
enumerable: true,
|
|
29
23
|
configurable: true,
|
|
30
24
|
writable: true,
|
|
31
25
|
value: void 0
|
|
32
26
|
});
|
|
33
|
-
Object.defineProperty(this, "producer", {
|
|
34
|
-
enumerable: true,
|
|
35
|
-
configurable: true,
|
|
36
|
-
writable: true,
|
|
37
|
-
value: null
|
|
38
|
-
});
|
|
39
27
|
Object.defineProperty(this, "topic", {
|
|
40
28
|
enumerable: true,
|
|
41
29
|
configurable: true,
|
|
42
30
|
writable: true,
|
|
43
31
|
value: void 0
|
|
44
32
|
});
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
33
|
+
this.kafkaProducer = new kafka_js_1.KafkaProducer({
|
|
34
|
+
producerName: config.producerName,
|
|
35
|
+
environment: config.environment,
|
|
36
|
+
shouldCompress: config.shouldCompress,
|
|
37
|
+
username: config.username,
|
|
38
|
+
password: config.password,
|
|
50
39
|
});
|
|
51
|
-
|
|
52
|
-
this.kafka = new kafkajs_1.Kafka({
|
|
53
|
-
clientId: `${producerName}-${environment}`,
|
|
54
|
-
brokers: environment === "production"
|
|
55
|
-
? ["warpstream.thirdweb.xyz:9092"]
|
|
56
|
-
: ["warpstream-dev.thirdweb.xyz:9092"],
|
|
57
|
-
ssl: {
|
|
58
|
-
checkServerIdentity(hostname, cert) {
|
|
59
|
-
return (0, node_tls_1.checkServerIdentity)(hostname.toLowerCase(), cert);
|
|
60
|
-
},
|
|
61
|
-
},
|
|
62
|
-
sasl: {
|
|
63
|
-
mechanism: "plain",
|
|
64
|
-
username,
|
|
65
|
-
password,
|
|
66
|
-
},
|
|
67
|
-
});
|
|
68
|
-
this.topic = (0, usageV2_js_1.getTopicName)(source);
|
|
69
|
-
this.compression = shouldCompress
|
|
70
|
-
? kafkajs_1.CompressionTypes.LZ4
|
|
71
|
-
: kafkajs_1.CompressionTypes.None;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Connect the producer.
|
|
75
|
-
* This must be called before calling `sendEvents()`.
|
|
76
|
-
*/
|
|
77
|
-
async init(configOverrides) {
|
|
78
|
-
if (this.compression === kafkajs_1.CompressionTypes.LZ4) {
|
|
79
|
-
CompressionCodecs[kafkajs_1.CompressionTypes.LZ4] = () => ({
|
|
80
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
81
|
-
compress: (encoder) => {
|
|
82
|
-
const compressed = (0, lz4js_1.compress)(encoder.buffer);
|
|
83
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
84
|
-
return Buffer.from(compressed);
|
|
85
|
-
},
|
|
86
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
87
|
-
decompress: (buffer) => {
|
|
88
|
-
const decompressed = (0, lz4js_1.decompress)(buffer);
|
|
89
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
90
|
-
return Buffer.from(decompressed);
|
|
91
|
-
},
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
this.producer = this.kafka.producer({
|
|
95
|
-
allowAutoTopicCreation: false,
|
|
96
|
-
...configOverrides,
|
|
97
|
-
});
|
|
98
|
-
await this.producer.connect();
|
|
40
|
+
this.topic = (0, usageV2_js_1.getTopicName)(config.source);
|
|
99
41
|
}
|
|
100
42
|
/**
|
|
101
43
|
* Send usageV2 events.
|
|
102
44
|
* This method may throw. To call this non-blocking:
|
|
103
|
-
*
|
|
104
|
-
* ```ts
|
|
105
|
-
* usageV2 = new UsageV2Producer(...)
|
|
106
|
-
* void usageV2.sendEvents(events).catch(console.error)
|
|
107
|
-
*
|
|
108
|
-
* @param events - The events to send.
|
|
45
|
+
* @param events
|
|
109
46
|
*/
|
|
110
47
|
async sendEvents(events,
|
|
111
48
|
/**
|
|
112
49
|
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
113
50
|
*/
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
? event.team_id.slice(5)
|
|
126
|
-
: event.team_id,
|
|
127
|
-
};
|
|
128
|
-
});
|
|
129
|
-
await this.producer.send({
|
|
130
|
-
topic: this.topic,
|
|
131
|
-
messages: parsedEvents.map((event) => ({
|
|
132
|
-
value: JSON.stringify(event),
|
|
133
|
-
})),
|
|
134
|
-
acks: -1, // All brokers must acknowledge
|
|
135
|
-
timeout: 10_000, // 10 seconds
|
|
136
|
-
compression: this.compression,
|
|
137
|
-
...configOverrides,
|
|
138
|
-
});
|
|
51
|
+
options) {
|
|
52
|
+
const parsedEvents = events.map((event) => ({
|
|
53
|
+
...event,
|
|
54
|
+
id: event.id ?? (0, node_crypto_1.randomUUID)(),
|
|
55
|
+
created_at: event.created_at ?? new Date(),
|
|
56
|
+
// Remove the "team_" prefix, if any.
|
|
57
|
+
team_id: event.team_id.startsWith("team_")
|
|
58
|
+
? event.team_id.slice(5)
|
|
59
|
+
: event.team_id,
|
|
60
|
+
}));
|
|
61
|
+
await this.kafkaProducer.send(this.topic, parsedEvents, options);
|
|
139
62
|
}
|
|
140
63
|
/**
|
|
141
64
|
* Disconnects UsageV2Producer.
|
|
142
65
|
* Useful when shutting down the service to flush in-flight events.
|
|
143
66
|
*/
|
|
144
67
|
async disconnect() {
|
|
145
|
-
|
|
146
|
-
await this.producer.disconnect();
|
|
147
|
-
this.producer = null;
|
|
148
|
-
}
|
|
68
|
+
await this.kafkaProducer.disconnect();
|
|
149
69
|
}
|
|
150
70
|
}
|
|
151
71
|
exports.UsageV2Producer = UsageV2Producer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usageV2.js","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":";;;AAAA,6CAAyC;AACzC,
|
|
1
|
+
{"version":3,"file":"usageV2.js","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":";;;AAAA,6CAAyC;AACzC,mDAI4B;AAC5B,yCAA2C;AAE3C;;;;;;;;;;;GAWG;AACH,MAAa,eAAe;IAI1B,YAAY,MAoBX;QAvBO;;;;;WAA6B;QAC7B;;;;;WAAc;QAuBpB,IAAI,CAAC,aAAa,GAAG,IAAI,wBAAa,CAAC;YACrC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,IAAA,yBAAY,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CACd,MAAsB;IACtB;;OAEG;IACH,OAIC;QAED,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,KAAK;YACR,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,IAAA,wBAAU,GAAE;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI,IAAI,EAAE;YAC1C,qCAAqC;YACrC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;gBACxC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBACxB,CAAC,CAAC,KAAK,CAAC,OAAO;SAClB,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;IACxC,CAAC;CACF;AAtED,0CAsEC"}
|
package/dist/esm/node/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAQvD,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAQpC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAoB,EACpB,aAAgC,EAChC,YAA2B;IAE3B,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,QAAQ,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;YACvD,OAAO;gBACL,UAAU,EAAE,KAAK;gBACjB,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,iDAAiD;gBAC/D,SAAS,EAAE,cAAc;aAC1B,CAAC;QACJ,CAAC;QACD,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,MAAM,EAAE,GAAG;YACX,YAAY,EAAE,uBAAuB;YACrC,SAAS,EAAE,uBAAuB;SACnC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,SAAS,CAAC,QAAQ,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CACpB,OAAsC;IAEtC,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAChB,OAAsC,EACtC,UAAkB;IAElB,OAAO,aAAa,CAAC,OAAO,CAAC;QAC3B,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACpC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;QACjC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,SAAoB;IAEpB,IAAI,UAAe,CAAC;IAEpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,GAAG,CAClB,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EACvB,UAAU,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC;IACtC,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACrD,+CAA+C;IAC/C,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE1C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,0CAA0C;QAC1C,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,4CAA4C;IAC5C,IAAI,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEjD,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,WAAW,GAAG,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACvD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,kBAAkB,GAAG,SAAS,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACtE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,kBAAkB,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC1C,iEAAiE;IACjE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IACD,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;QAChC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,SAAS,EAAE,CAAC;QACd,sBAAsB;QACtB,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,GAAG,GAAkB,IAAI,CAAC;IAC9B,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,gDAAgD;IAChD,MAAM,mBAAmB,GAAG,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,mBAAmB,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAChD,GAAG,GAAG,KAAK,CAAC;YACZ,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;YAClE,sFAAsF;YACtF,IAAI,gBAAgB,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBAC/C,aAAa,GAAG,gBAAgB,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,GAAG;QACH,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1C,aAAa;QACb,SAAS;QACT,QAAQ;QACR,WAAW;QACX,kBAAkB;QAClB,MAAM;QACN,QAAQ;QACR,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,aAAa;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAC7B,QAAQ,EACR,GAAG,EACH,GAAG,EACH,QAAQ,EACR,aAAa,EACb,SAAS,GAQV;IACC,IAAI,CAAC;QACH,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,GAAG;YACjB,YAAY,EAAE,CAAC,CAAC,iBAAiB,CAAC,SAAS;YAC3C,WAAW,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ;YACzC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG;YAC/B,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,GAAG,CAAC,UAAU;YACtB,aAAa;YACb,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;YACzC,UAAU,EAAE,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC;YAC/C,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC;YAC9C,EAAE,EAAE,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC;YAClC,SAAS;SACV,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAOzC,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAQvD,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAQpC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAoB,EACpB,aAAgC,EAChC,YAA2B;IAE3B,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,QAAQ,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;YACvD,OAAO;gBACL,UAAU,EAAE,KAAK;gBACjB,MAAM,EAAE,GAAG;gBACX,YAAY,EAAE,iDAAiD;gBAC/D,SAAS,EAAE,cAAc;aAC1B,CAAC;QACJ,CAAC;QACD,OAAO;YACL,UAAU,EAAE,KAAK;YACjB,MAAM,EAAE,GAAG;YACX,YAAY,EAAE,uBAAuB;YACrC,SAAS,EAAE,uBAAuB;SACnC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,SAAS,CAAC,QAAQ,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CACpB,OAAsC;IAEtC,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,SAAS,CAChB,OAAsC,EACtC,UAAkB;IAElB,OAAO,aAAa,CAAC,OAAO,CAAC;QAC3B,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACpC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC;QACjC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,SAAoB;IAEpB,IAAI,UAAe,CAAC;IAEpB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,GAAG,CAClB,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EACvB,UAAU,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,CACrD,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC;IACtC,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACrD,+CAA+C;IAC/C,IAAI,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE1C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,0CAA0C;QAC1C,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC/C,CAAC;IAED,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,4CAA4C;IAC5C,IAAI,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAEjD,kCAAkC;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,WAAW,GAAG,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACvD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,kBAAkB,GAAG,SAAS,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAC;IACtE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,kBAAkB,GAAG,UAAU,CAAC,YAAY,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;IACzE,CAAC;IAED,IAAI,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAC1C,iEAAiE;IACjE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IACD,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC;QAChC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,wBAAwB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,SAAS,EAAE,CAAC;QACd,sBAAsB;QACtB,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,GAAG,GAAkB,IAAI,CAAC;IAC9B,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,gDAAgD;IAChD,MAAM,mBAAmB,GAAG,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,IAAI,mBAAmB,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,mBAAmB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,IAAI,EAAE,WAAW,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YAChD,GAAG,GAAG,KAAK,CAAC;YACZ,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,EAAE,oBAAoB,CAAC,CAAC;YAClE,sFAAsF;YACtF,IAAI,gBAAgB,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBAC/C,aAAa,GAAG,gBAAgB,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,GAAG;QACH,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1C,aAAa;QACb,SAAS;QACT,QAAQ;QACR,WAAW;QACX,kBAAkB;QAClB,MAAM;QACN,QAAQ;QACR,aAAa,EAAE,SAAS,CAAC,aAAa;QACtC,aAAa;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,SAAiB;IAC7C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,EAC7B,QAAQ,EACR,GAAG,EACH,GAAG,EACH,QAAQ,EACR,aAAa,EACb,SAAS,GAQV;IACC,IAAI,CAAC;QACH,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;QAE5B,OAAO,CAAC,GAAG,CACT,IAAI,CAAC,SAAS,CAAC;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,GAAG;YACjB,YAAY,EAAE,CAAC,CAAC,iBAAiB,CAAC,SAAS;YAC3C,WAAW,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ;YACzC,MAAM,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG;YAC/B,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;YACpC,QAAQ;YACR,MAAM,EAAE,GAAG,CAAC,UAAU;YACtB,aAAa;YACb,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,CAAC;YACzC,UAAU,EAAE,SAAS,CAAC,OAAO,EAAE,eAAe,CAAC;YAC/C,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,gBAAgB,CAAC;YAC9C,EAAE,EAAE,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC;YAClC,SAAS;SACV,CAAC,CACH,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { checkServerIdentity } from "node:tls";
|
|
2
|
+
import { CompressionTypes, Kafka } from "kafkajs";
|
|
3
|
+
import { compress, decompress } from "lz4js";
|
|
4
|
+
// CompressionCodecs is not exported properly in kafkajs. Source: https://github.com/tulios/kafkajs/issues/1391
|
|
5
|
+
import KafkaJS from "kafkajs";
|
|
6
|
+
const { CompressionCodecs } = KafkaJS;
|
|
7
|
+
/**
|
|
8
|
+
* Creates a KafkaProducer which opens a persistent TCP connection.
|
|
9
|
+
* This class is thread-safe so your service should re-use one instance.
|
|
10
|
+
*
|
|
11
|
+
* Example:
|
|
12
|
+
* ```ts
|
|
13
|
+
* kafka = new KafkaProducer(...)
|
|
14
|
+
* await kafka.send(topic, events)
|
|
15
|
+
* // Non-blocking:
|
|
16
|
+
* // void kafka.send(topic, events).catch((e) => console.error(e))
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export class KafkaProducer {
|
|
20
|
+
constructor(config) {
|
|
21
|
+
Object.defineProperty(this, "kafka", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
27
|
+
Object.defineProperty(this, "producer", {
|
|
28
|
+
enumerable: true,
|
|
29
|
+
configurable: true,
|
|
30
|
+
writable: true,
|
|
31
|
+
value: null
|
|
32
|
+
});
|
|
33
|
+
Object.defineProperty(this, "compression", {
|
|
34
|
+
enumerable: true,
|
|
35
|
+
configurable: true,
|
|
36
|
+
writable: true,
|
|
37
|
+
value: void 0
|
|
38
|
+
});
|
|
39
|
+
const { producerName, environment, shouldCompress = true, username, password, } = config;
|
|
40
|
+
this.kafka = new Kafka({
|
|
41
|
+
clientId: `${producerName}-${environment}`,
|
|
42
|
+
brokers: environment === "production"
|
|
43
|
+
? ["warpstream.thirdweb.xyz:9092"]
|
|
44
|
+
: ["warpstream-dev.thirdweb.xyz:9092"],
|
|
45
|
+
ssl: {
|
|
46
|
+
checkServerIdentity(hostname, cert) {
|
|
47
|
+
return checkServerIdentity(hostname.toLowerCase(), cert);
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
sasl: {
|
|
51
|
+
mechanism: "plain",
|
|
52
|
+
username,
|
|
53
|
+
password,
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
if (shouldCompress) {
|
|
57
|
+
this.compression = CompressionTypes.LZ4;
|
|
58
|
+
CompressionCodecs[CompressionTypes.LZ4] = () => ({
|
|
59
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
60
|
+
compress: (encoder) => {
|
|
61
|
+
const compressed = compress(encoder.buffer);
|
|
62
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
63
|
+
return Buffer.from(compressed);
|
|
64
|
+
},
|
|
65
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
66
|
+
decompress: (buffer) => {
|
|
67
|
+
const decompressed = decompress(buffer);
|
|
68
|
+
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
69
|
+
return Buffer.from(decompressed);
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
this.compression = CompressionTypes.None;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Send messages to a Kafka topic.
|
|
79
|
+
* This method may throw. To call this non-blocking:
|
|
80
|
+
* @param topic
|
|
81
|
+
* @param messages
|
|
82
|
+
* @param configOverrides
|
|
83
|
+
*/
|
|
84
|
+
async send(topic, messages,
|
|
85
|
+
/**
|
|
86
|
+
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
87
|
+
*/
|
|
88
|
+
options) {
|
|
89
|
+
if (!this.producer) {
|
|
90
|
+
this.producer = this.kafka.producer({
|
|
91
|
+
allowAutoTopicCreation: options?.allowAutoTopicCreation ?? false,
|
|
92
|
+
});
|
|
93
|
+
await this.producer.connect();
|
|
94
|
+
}
|
|
95
|
+
await this.producer.send({
|
|
96
|
+
topic,
|
|
97
|
+
messages: messages.map((m) => ({
|
|
98
|
+
value: JSON.stringify(m),
|
|
99
|
+
})),
|
|
100
|
+
compression: this.compression,
|
|
101
|
+
acks: options?.acks ?? -1, // Default: All brokers must acknowledge
|
|
102
|
+
timeout: options?.timeout ?? 10_000, // Default: 10 seconds
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Disconnects KafkaProducer.
|
|
107
|
+
* Useful when shutting down the service to flush in-flight events.
|
|
108
|
+
*/
|
|
109
|
+
async disconnect() {
|
|
110
|
+
if (this.producer) {
|
|
111
|
+
await this.producer.disconnect();
|
|
112
|
+
this.producer = null;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=kafka.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../../src/node/kafka.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,KAAK,EAAiB,MAAM,SAAS,CAAC;AACjE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AAE7C,+GAA+G;AAC/G,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,MAAM,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,aAAa;IAKxB,YAAY,MAgBX;QApBO;;;;;WAAa;QACb;;;;mBAA4B,IAAI;WAAC;QACjC;;;;;WAA8B;QAmBpC,MAAM,EACJ,YAAY,EACZ,WAAW,EACX,cAAc,GAAG,IAAI,EACrB,QAAQ,EACR,QAAQ,GACT,GAAG,MAAM,CAAC;QAEX,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC;YACrB,QAAQ,EAAE,GAAG,YAAY,IAAI,WAAW,EAAE;YAC1C,OAAO,EACL,WAAW,KAAK,YAAY;gBAC1B,CAAC,CAAC,CAAC,8BAA8B,CAAC;gBAClC,CAAC,CAAC,CAAC,kCAAkC,CAAC;YAC1C,GAAG,EAAE;gBACH,mBAAmB,CAAC,QAAQ,EAAE,IAAI;oBAChC,OAAO,mBAAmB,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,IAAI,CAAC,CAAC;gBAC3D,CAAC;aACF;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,OAAO;gBAClB,QAAQ;gBACR,QAAQ;aACT;SACF,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACnB,IAAI,CAAC,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC;YAExC,iBAAiB,CAAC,gBAAgB,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;gBAC/C,wEAAwE;gBACxE,QAAQ,EAAE,CAAC,OAA2B,EAAE,EAAE;oBACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBAC5C,wEAAwE;oBACxE,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACjC,CAAC;gBACD,wEAAwE;gBACxE,UAAU,EAAE,CAAC,MAAc,EAAE,EAAE;oBAC7B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;oBACxC,wEAAwE;oBACxE,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACnC,CAAC;aACF,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC;QAC3C,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CACR,KAAa,EACb,QAAmC;IACnC;;OAEG;IACH,OAIC;QAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;gBAClC,sBAAsB,EAAE,OAAO,EAAE,sBAAsB,IAAI,KAAK;aACjE,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QAChC,CAAC;QAED,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACvB,KAAK;YACL,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC7B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;aACzB,CAAC,CAAC;YACH,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,EAAE,wCAAwC;YACnE,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,MAAM,EAAE,sBAAsB;SAC5D,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;CACF"}
|
package/dist/esm/node/usageV2.js
CHANGED
|
@@ -1,11 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { checkServerIdentity } from "node:tls";
|
|
3
|
-
import { CompressionTypes, Kafka, } from "kafkajs";
|
|
4
|
-
import { compress, decompress } from "lz4js";
|
|
5
2
|
import { getTopicName, } from "../core/usageV2.js";
|
|
6
|
-
|
|
7
|
-
import KafkaJS from "kafkajs";
|
|
8
|
-
const { CompressionCodecs } = KafkaJS;
|
|
3
|
+
import { KafkaProducer } from "./kafka.js";
|
|
9
4
|
/**
|
|
10
5
|
* Creates a UsageV2Producer which opens a persistent TCP connection.
|
|
11
6
|
* This class is thread-safe so your service should re-use one instance.
|
|
@@ -13,136 +8,61 @@ const { CompressionCodecs } = KafkaJS;
|
|
|
13
8
|
* Example:
|
|
14
9
|
* ```ts
|
|
15
10
|
* usageV2 = new UsageV2Producer(..)
|
|
16
|
-
* await usageV2.init()
|
|
17
11
|
* await usageV2.sendEvents(events)
|
|
18
12
|
* // Non-blocking:
|
|
19
|
-
* // void usageV2.sendEvents(events).catch(console.error)
|
|
13
|
+
* // void usageV2.sendEvents(events).catch((e) => console.error(e))
|
|
20
14
|
* ```
|
|
21
15
|
*/
|
|
22
16
|
export class UsageV2Producer {
|
|
23
17
|
constructor(config) {
|
|
24
|
-
Object.defineProperty(this, "
|
|
18
|
+
Object.defineProperty(this, "kafkaProducer", {
|
|
25
19
|
enumerable: true,
|
|
26
20
|
configurable: true,
|
|
27
21
|
writable: true,
|
|
28
22
|
value: void 0
|
|
29
23
|
});
|
|
30
|
-
Object.defineProperty(this, "producer", {
|
|
31
|
-
enumerable: true,
|
|
32
|
-
configurable: true,
|
|
33
|
-
writable: true,
|
|
34
|
-
value: null
|
|
35
|
-
});
|
|
36
24
|
Object.defineProperty(this, "topic", {
|
|
37
25
|
enumerable: true,
|
|
38
26
|
configurable: true,
|
|
39
27
|
writable: true,
|
|
40
28
|
value: void 0
|
|
41
29
|
});
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
30
|
+
this.kafkaProducer = new KafkaProducer({
|
|
31
|
+
producerName: config.producerName,
|
|
32
|
+
environment: config.environment,
|
|
33
|
+
shouldCompress: config.shouldCompress,
|
|
34
|
+
username: config.username,
|
|
35
|
+
password: config.password,
|
|
47
36
|
});
|
|
48
|
-
|
|
49
|
-
this.kafka = new Kafka({
|
|
50
|
-
clientId: `${producerName}-${environment}`,
|
|
51
|
-
brokers: environment === "production"
|
|
52
|
-
? ["warpstream.thirdweb.xyz:9092"]
|
|
53
|
-
: ["warpstream-dev.thirdweb.xyz:9092"],
|
|
54
|
-
ssl: {
|
|
55
|
-
checkServerIdentity(hostname, cert) {
|
|
56
|
-
return checkServerIdentity(hostname.toLowerCase(), cert);
|
|
57
|
-
},
|
|
58
|
-
},
|
|
59
|
-
sasl: {
|
|
60
|
-
mechanism: "plain",
|
|
61
|
-
username,
|
|
62
|
-
password,
|
|
63
|
-
},
|
|
64
|
-
});
|
|
65
|
-
this.topic = getTopicName(source);
|
|
66
|
-
this.compression = shouldCompress
|
|
67
|
-
? CompressionTypes.LZ4
|
|
68
|
-
: CompressionTypes.None;
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Connect the producer.
|
|
72
|
-
* This must be called before calling `sendEvents()`.
|
|
73
|
-
*/
|
|
74
|
-
async init(configOverrides) {
|
|
75
|
-
if (this.compression === CompressionTypes.LZ4) {
|
|
76
|
-
CompressionCodecs[CompressionTypes.LZ4] = () => ({
|
|
77
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
78
|
-
compress: (encoder) => {
|
|
79
|
-
const compressed = compress(encoder.buffer);
|
|
80
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
81
|
-
return Buffer.from(compressed);
|
|
82
|
-
},
|
|
83
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
84
|
-
decompress: (buffer) => {
|
|
85
|
-
const decompressed = decompress(buffer);
|
|
86
|
-
// biome-ignore lint/style/noRestrictedGlobals: kafkajs expects a Buffer
|
|
87
|
-
return Buffer.from(decompressed);
|
|
88
|
-
},
|
|
89
|
-
});
|
|
90
|
-
}
|
|
91
|
-
this.producer = this.kafka.producer({
|
|
92
|
-
allowAutoTopicCreation: false,
|
|
93
|
-
...configOverrides,
|
|
94
|
-
});
|
|
95
|
-
await this.producer.connect();
|
|
37
|
+
this.topic = getTopicName(config.source);
|
|
96
38
|
}
|
|
97
39
|
/**
|
|
98
40
|
* Send usageV2 events.
|
|
99
41
|
* This method may throw. To call this non-blocking:
|
|
100
|
-
*
|
|
101
|
-
* ```ts
|
|
102
|
-
* usageV2 = new UsageV2Producer(...)
|
|
103
|
-
* void usageV2.sendEvents(events).catch(console.error)
|
|
104
|
-
*
|
|
105
|
-
* @param events - The events to send.
|
|
42
|
+
* @param events
|
|
106
43
|
*/
|
|
107
44
|
async sendEvents(events,
|
|
108
45
|
/**
|
|
109
46
|
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
110
47
|
*/
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
? event.team_id.slice(5)
|
|
123
|
-
: event.team_id,
|
|
124
|
-
};
|
|
125
|
-
});
|
|
126
|
-
await this.producer.send({
|
|
127
|
-
topic: this.topic,
|
|
128
|
-
messages: parsedEvents.map((event) => ({
|
|
129
|
-
value: JSON.stringify(event),
|
|
130
|
-
})),
|
|
131
|
-
acks: -1, // All brokers must acknowledge
|
|
132
|
-
timeout: 10_000, // 10 seconds
|
|
133
|
-
compression: this.compression,
|
|
134
|
-
...configOverrides,
|
|
135
|
-
});
|
|
48
|
+
options) {
|
|
49
|
+
const parsedEvents = events.map((event) => ({
|
|
50
|
+
...event,
|
|
51
|
+
id: event.id ?? randomUUID(),
|
|
52
|
+
created_at: event.created_at ?? new Date(),
|
|
53
|
+
// Remove the "team_" prefix, if any.
|
|
54
|
+
team_id: event.team_id.startsWith("team_")
|
|
55
|
+
? event.team_id.slice(5)
|
|
56
|
+
: event.team_id,
|
|
57
|
+
}));
|
|
58
|
+
await this.kafkaProducer.send(this.topic, parsedEvents, options);
|
|
136
59
|
}
|
|
137
60
|
/**
|
|
138
61
|
* Disconnects UsageV2Producer.
|
|
139
62
|
* Useful when shutting down the service to flush in-flight events.
|
|
140
63
|
*/
|
|
141
64
|
async disconnect() {
|
|
142
|
-
|
|
143
|
-
await this.producer.disconnect();
|
|
144
|
-
this.producer = null;
|
|
145
|
-
}
|
|
65
|
+
await this.kafkaProducer.disconnect();
|
|
146
66
|
}
|
|
147
67
|
}
|
|
148
68
|
//# sourceMappingURL=usageV2.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usageV2.js","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,
|
|
1
|
+
{"version":3,"file":"usageV2.js","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAGL,YAAY,GACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,eAAe;IAI1B,YAAY,MAoBX;QAvBO;;;;;WAA6B;QAC7B;;;;;WAAc;QAuBpB,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC;YACrC,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CACd,MAAsB;IACtB;;OAEG;IACH,OAIC;QAED,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC1C,GAAG,KAAK;YACR,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,UAAU,EAAE;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI,IAAI,EAAE;YAC1C,qCAAqC;YACrC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;gBACxC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBACxB,CAAC,CAAC,KAAK,CAAC,OAAO;SAClB,CAAC,CAAC,CAAC;QACJ,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;IACxC,CAAC;CACF"}
|
|
@@ -3,6 +3,7 @@ import type { CoreServiceConfig } from "../core/api.js";
|
|
|
3
3
|
import type { AuthorizationInput, CacheOptions } from "../core/authorize/index.js";
|
|
4
4
|
import type { AuthorizationResult } from "../core/authorize/types.js";
|
|
5
5
|
import type { CoreAuthInput } from "../core/types.js";
|
|
6
|
+
export * from "./kafka.js";
|
|
6
7
|
export * from "./usageV2.js";
|
|
7
8
|
export * from "../core/usage.js";
|
|
8
9
|
export * from "../core/usageV2.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,eAAe,EACf,cAAc,EACf,MAAM,WAAW,CAAC;AACnB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EACV,kBAAkB,EAClB,YAAY,EACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAEpC,KAAK,iBAAiB,GAAG,iBAAiB,CAAC;AAE3C,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG;IACtC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC;CAChC,CAAC;AAEF,wBAAsB,aAAa,CACjC,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,iBAAiB,EAChC,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,mBAAmB,CAAC,CAsB9B;AAmBD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,SAAS,GACnB,kBAAkB,CAgGpB;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,UAE9C;AAED,wBAAgB,cAAc,CAAC,EAC7B,QAAQ,EACR,GAAG,EACH,GAAG,EACH,QAAQ,EACR,aAAa,EACb,SAAS,GACV,EAAE,SAAS,GAAG;IAEb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,cAAc,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,QAwBA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/node/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,eAAe,EACf,cAAc,EACf,MAAM,WAAW,CAAC;AACnB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EACV,kBAAkB,EAClB,YAAY,EACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,kBAAkB,CAAC;AACjC,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AAEpC,KAAK,iBAAiB,GAAG,iBAAiB,CAAC;AAE3C,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG;IACtC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC;CAChC,CAAC;AAEF,wBAAsB,aAAa,CACjC,SAAS,EAAE,SAAS,EACpB,aAAa,EAAE,iBAAiB,EAChC,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,mBAAmB,CAAC,CAsB9B;AAmBD,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,SAAS,GACnB,kBAAkB,CAgGpB;AAED,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,UAE9C;AAED,wBAAgB,cAAc,CAAC,EAC7B,QAAQ,EACR,GAAG,EACH,GAAG,EACH,QAAQ,EACR,aAAa,EACb,SAAS,GACV,EAAE,SAAS,GAAG;IAEb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,cAAc,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,QAwBA"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a KafkaProducer which opens a persistent TCP connection.
|
|
3
|
+
* This class is thread-safe so your service should re-use one instance.
|
|
4
|
+
*
|
|
5
|
+
* Example:
|
|
6
|
+
* ```ts
|
|
7
|
+
* kafka = new KafkaProducer(...)
|
|
8
|
+
* await kafka.send(topic, events)
|
|
9
|
+
* // Non-blocking:
|
|
10
|
+
* // void kafka.send(topic, events).catch((e) => console.error(e))
|
|
11
|
+
* ```
|
|
12
|
+
*/
|
|
13
|
+
export declare class KafkaProducer {
|
|
14
|
+
private kafka;
|
|
15
|
+
private producer;
|
|
16
|
+
private compression;
|
|
17
|
+
constructor(config: {
|
|
18
|
+
/**
|
|
19
|
+
* A descriptive name for your service. Example: "storage-server"
|
|
20
|
+
*/
|
|
21
|
+
producerName: string;
|
|
22
|
+
/**
|
|
23
|
+
* The environment the service is running in.
|
|
24
|
+
*/
|
|
25
|
+
environment: "development" | "production";
|
|
26
|
+
/**
|
|
27
|
+
* Whether to compress the events.
|
|
28
|
+
*/
|
|
29
|
+
shouldCompress?: boolean;
|
|
30
|
+
username: string;
|
|
31
|
+
password: string;
|
|
32
|
+
});
|
|
33
|
+
/**
|
|
34
|
+
* Send messages to a Kafka topic.
|
|
35
|
+
* This method may throw. To call this non-blocking:
|
|
36
|
+
* @param topic
|
|
37
|
+
* @param messages
|
|
38
|
+
* @param configOverrides
|
|
39
|
+
*/
|
|
40
|
+
send(topic: string, messages: Record<string, unknown>[],
|
|
41
|
+
/**
|
|
42
|
+
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
43
|
+
*/
|
|
44
|
+
options?: {
|
|
45
|
+
acks?: number;
|
|
46
|
+
timeout?: number;
|
|
47
|
+
allowAutoTopicCreation?: boolean;
|
|
48
|
+
}): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Disconnects KafkaProducer.
|
|
51
|
+
* Useful when shutting down the service to flush in-flight events.
|
|
52
|
+
*/
|
|
53
|
+
disconnect(): Promise<void>;
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=kafka.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"kafka.d.ts","sourceRoot":"","sources":["../../../src/node/kafka.ts"],"names":[],"mappings":"AAQA;;;;;;;;;;;GAWG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,WAAW,CAAmB;gBAE1B,MAAM,EAAE;QAClB;;WAEG;QACH,YAAY,EAAE,MAAM,CAAC;QACrB;;WAEG;QACH,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;QAC1C;;WAEG;QACH,cAAc,CAAC,EAAE,OAAO,CAAC;QAEzB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAiDD;;;;;;OAMG;IACG,IAAI,CACR,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE;IACnC;;OAEG;IACH,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,sBAAsB,CAAC,EAAE,OAAO,CAAC;KAClC,GACA,OAAO,CAAC,IAAI,CAAC;IAmBhB;;;OAGG;IACG,UAAU;CAMjB"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { type ProducerConfig } from "kafkajs";
|
|
2
1
|
import { type UsageV2Event, type UsageV2Source } from "../core/usageV2.js";
|
|
3
2
|
/**
|
|
4
3
|
* Creates a UsageV2Producer which opens a persistent TCP connection.
|
|
@@ -7,17 +6,14 @@ import { type UsageV2Event, type UsageV2Source } from "../core/usageV2.js";
|
|
|
7
6
|
* Example:
|
|
8
7
|
* ```ts
|
|
9
8
|
* usageV2 = new UsageV2Producer(..)
|
|
10
|
-
* await usageV2.init()
|
|
11
9
|
* await usageV2.sendEvents(events)
|
|
12
10
|
* // Non-blocking:
|
|
13
|
-
* // void usageV2.sendEvents(events).catch(console.error)
|
|
11
|
+
* // void usageV2.sendEvents(events).catch((e) => console.error(e))
|
|
14
12
|
* ```
|
|
15
13
|
*/
|
|
16
14
|
export declare class UsageV2Producer {
|
|
17
|
-
private
|
|
18
|
-
private producer;
|
|
15
|
+
private kafkaProducer;
|
|
19
16
|
private topic;
|
|
20
|
-
private compression;
|
|
21
17
|
constructor(config: {
|
|
22
18
|
/**
|
|
23
19
|
* A descriptive name for your service. Example: "storage-server"
|
|
@@ -38,28 +34,19 @@ export declare class UsageV2Producer {
|
|
|
38
34
|
username: string;
|
|
39
35
|
password: string;
|
|
40
36
|
});
|
|
41
|
-
/**
|
|
42
|
-
* Connect the producer.
|
|
43
|
-
* This must be called before calling `sendEvents()`.
|
|
44
|
-
*/
|
|
45
|
-
init(configOverrides?: ProducerConfig): Promise<void>;
|
|
46
37
|
/**
|
|
47
38
|
* Send usageV2 events.
|
|
48
39
|
* This method may throw. To call this non-blocking:
|
|
49
|
-
*
|
|
50
|
-
* ```ts
|
|
51
|
-
* usageV2 = new UsageV2Producer(...)
|
|
52
|
-
* void usageV2.sendEvents(events).catch(console.error)
|
|
53
|
-
*
|
|
54
|
-
* @param events - The events to send.
|
|
40
|
+
* @param events
|
|
55
41
|
*/
|
|
56
42
|
sendEvents(events: UsageV2Event[],
|
|
57
43
|
/**
|
|
58
44
|
* Reference: https://kafka.js.org/docs/producing#producing-messages
|
|
59
45
|
*/
|
|
60
|
-
|
|
46
|
+
options?: {
|
|
61
47
|
acks?: number;
|
|
62
48
|
timeout?: number;
|
|
49
|
+
allowAutoTopicCreation?: boolean;
|
|
63
50
|
}): Promise<void>;
|
|
64
51
|
/**
|
|
65
52
|
* Disconnects UsageV2Producer.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"usageV2.d.ts","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"usageV2.d.ts","sourceRoot":"","sources":["../../../src/node/usageV2.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,YAAY,EACjB,KAAK,aAAa,EAEnB,MAAM,oBAAoB,CAAC;AAG5B;;;;;;;;;;;GAWG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE;QAClB;;WAEG;QACH,YAAY,EAAE,MAAM,CAAC;QACrB;;WAEG;QACH,WAAW,EAAE,aAAa,GAAG,YAAY,CAAC;QAC1C;;WAEG;QACH,MAAM,EAAE,aAAa,CAAC;QACtB;;WAEG;QACH,cAAc,CAAC,EAAE,OAAO,CAAC;QAEzB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB;IAWD;;;;OAIG;IACG,UAAU,CACd,MAAM,EAAE,YAAY,EAAE;IACtB;;OAEG;IACH,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,sBAAsB,CAAC,EAAE,OAAO,CAAC;KAClC,GACA,OAAO,CAAC,IAAI,CAAC;IAahB;;;OAGG;IACG,UAAU;CAGjB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thirdweb-dev/service-utils",
|
|
3
|
-
"version": "0.8.5-nightly-
|
|
3
|
+
"version": "0.8.5-nightly-ee6bd4f8982d2b0db823c8cb472aeb9d6f9408fb-20250205060502",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/cjs/index.js",
|
|
6
6
|
"module": "dist/esm/index.js",
|