@query-farm/vgi-rpc 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +47 -0
  2. package/dist/client/connect.d.ts.map +1 -1
  3. package/dist/client/index.d.ts +1 -1
  4. package/dist/client/index.d.ts.map +1 -1
  5. package/dist/client/oauth.d.ts +36 -0
  6. package/dist/client/oauth.d.ts.map +1 -1
  7. package/dist/client/pipe.d.ts +3 -0
  8. package/dist/client/pipe.d.ts.map +1 -1
  9. package/dist/client/stream.d.ts +3 -0
  10. package/dist/client/stream.d.ts.map +1 -1
  11. package/dist/client/types.d.ts +4 -0
  12. package/dist/client/types.d.ts.map +1 -1
  13. package/dist/constants.d.ts +3 -1
  14. package/dist/constants.d.ts.map +1 -1
  15. package/dist/dispatch/describe.d.ts.map +1 -1
  16. package/dist/dispatch/stream.d.ts +2 -1
  17. package/dist/dispatch/stream.d.ts.map +1 -1
  18. package/dist/dispatch/unary.d.ts +2 -1
  19. package/dist/dispatch/unary.d.ts.map +1 -1
  20. package/dist/external.d.ts +45 -0
  21. package/dist/external.d.ts.map +1 -0
  22. package/dist/gcs.d.ts +38 -0
  23. package/dist/gcs.d.ts.map +1 -0
  24. package/dist/http/auth.d.ts +13 -2
  25. package/dist/http/auth.d.ts.map +1 -1
  26. package/dist/http/bearer.d.ts +34 -0
  27. package/dist/http/bearer.d.ts.map +1 -0
  28. package/dist/http/dispatch.d.ts +2 -0
  29. package/dist/http/dispatch.d.ts.map +1 -1
  30. package/dist/http/handler.d.ts.map +1 -1
  31. package/dist/http/index.d.ts +4 -0
  32. package/dist/http/index.d.ts.map +1 -1
  33. package/dist/http/jwt.d.ts +2 -2
  34. package/dist/http/jwt.d.ts.map +1 -1
  35. package/dist/http/mtls.d.ts +78 -0
  36. package/dist/http/mtls.d.ts.map +1 -0
  37. package/dist/http/pages.d.ts +9 -0
  38. package/dist/http/pages.d.ts.map +1 -0
  39. package/dist/http/types.d.ts +17 -1
  40. package/dist/http/types.d.ts.map +1 -1
  41. package/dist/index.d.ts +3 -2
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/index.js +1119 -230
  44. package/dist/index.js.map +24 -20
  45. package/dist/otel.d.ts +47 -0
  46. package/dist/otel.d.ts.map +1 -0
  47. package/dist/s3.d.ts +43 -0
  48. package/dist/s3.d.ts.map +1 -0
  49. package/dist/server.d.ts +6 -0
  50. package/dist/server.d.ts.map +1 -1
  51. package/dist/types.d.ts +30 -0
  52. package/dist/types.d.ts.map +1 -1
  53. package/package.json +44 -1
  54. package/src/client/connect.ts +13 -5
  55. package/src/client/index.ts +10 -1
  56. package/src/client/introspect.ts +1 -1
  57. package/src/client/oauth.ts +94 -1
  58. package/src/client/pipe.ts +19 -4
  59. package/src/client/stream.ts +20 -7
  60. package/src/client/types.ts +4 -0
  61. package/src/constants.ts +4 -1
  62. package/src/dispatch/describe.ts +20 -0
  63. package/src/dispatch/stream.ts +7 -1
  64. package/src/dispatch/unary.ts +6 -1
  65. package/src/external.ts +209 -0
  66. package/src/gcs.ts +86 -0
  67. package/src/http/auth.ts +67 -4
  68. package/src/http/bearer.ts +107 -0
  69. package/src/http/dispatch.ts +26 -6
  70. package/src/http/handler.ts +81 -4
  71. package/src/http/index.ts +10 -0
  72. package/src/http/jwt.ts +17 -3
  73. package/src/http/mtls.ts +298 -0
  74. package/src/http/pages.ts +298 -0
  75. package/src/http/types.ts +17 -1
  76. package/src/index.ts +25 -0
  77. package/src/otel.ts +161 -0
  78. package/src/s3.ts +94 -0
  79. package/src/server.ts +42 -8
  80. package/src/types.ts +34 -0
package/src/http/pages.ts ADDED
@@ -0,0 +1,298 @@
1
+ // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
+ // SPDX-License-Identifier: Apache-2.0
3
+
4
+ /**
5
+ * Pre-rendered HTML pages for the vgi-rpc HTTP server.
6
+ * Matches the styling of the Python and Go implementations.
7
+ */
8
+
9
+ import type { MethodDefinition } from "../types.js";
10
+
11
+ const LOGO_URL = "https://vgi-rpc-python.query.farm/assets/logo-hero.png";
12
+
13
+ const FONTS = `<link rel="preconnect" href="https://fonts.googleapis.com">
14
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
15
+ <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&family=JetBrains+Mono:wght@400;600&display=swap" rel="stylesheet">`;
16
+
17
+ function escapeHtml(s: string): string {
18
+ return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
19
+ }
20
+
21
+ function arrowTypeToString(type: import("@query-farm/apache-arrow").DataType): string {
22
+ const id = type.typeId;
23
+ // Match the human-friendly type names used by the Python reference implementation
24
+ if (id === 5) return "str"; // Utf8
25
+ if (id === 4) return "bytes"; // Binary
26
+ if (id === 2) return "int"; // Int32/Int64
27
+ if (id === 3) return "float"; // Float32/Float64
28
+ if (id === 6) return "bool"; // Bool
29
+ if (id === 12) return "list"; // List
30
+ if (id === 17) return "map"; // Map
31
+ if (id === 24) return "enum"; // Dictionary
32
+ return type.toString();
33
+ }
34
+
35
+ // ---------------------------------------------------------------------------
36
+ // Landing page
37
+ // ---------------------------------------------------------------------------
38
+
39
+ export function buildLandingPage(
40
+ protocolName: string,
41
+ serverId: string,
42
+ describePath: string | null,
43
+ repoUrl: string | null,
44
+ ): string {
45
+ const links: string[] = [];
46
+ if (describePath) {
47
+ links.push(`<a class="primary" href="${escapeHtml(describePath)}">View service API</a>`);
48
+ }
49
+ if (repoUrl) {
50
+ links.push(`<a href="${escapeHtml(repoUrl)}">Source repository</a>`);
51
+ }
52
+ links.push(`<a href="https://vgi-rpc.query.farm">Learn more about <code>vgi-rpc</code></a>`);
53
+
54
+ return `<!DOCTYPE html>
55
+ <html lang="en">
56
+ <head>
57
+ <meta charset="utf-8">
58
+ <meta name="viewport" content="width=device-width, initial-scale=1">
59
+ <title>${escapeHtml(protocolName)} \u2014 vgi-rpc</title>
60
+ ${FONTS}
61
+ <style>
62
+ body { font-family: 'Inter', system-ui, -apple-system, sans-serif; max-width: 600px;
63
+ margin: 0 auto; padding: 60px 20px 0; color: #2c2c1e; text-align: center;
64
+ background: #faf8f0; }
65
+ .logo { margin-bottom: 24px; }
66
+ .logo img { width: 140px; height: 140px; border-radius: 50%;
67
+ box-shadow: 0 4px 24px rgba(0,0,0,0.12); }
68
+ h1 { color: #2d5016; margin-bottom: 8px; font-weight: 700; }
69
+ code { font-family: 'JetBrains Mono', monospace; background: #f0ece0;
70
+ padding: 2px 6px; border-radius: 3px; font-size: 0.9em; color: #2c2c1e; }
71
+ a { color: #2d5016; text-decoration: none; }
72
+ a:hover { color: #4a7c23; }
73
+ p { line-height: 1.7; color: #6b6b5a; }
74
+ .meta { font-size: 0.9em; color: #6b6b5a; }
75
+ .links { margin-top: 28px; display: flex; flex-wrap: wrap; justify-content: center; gap: 8px; }
76
+ .links a { display: inline-block; padding: 8px 18px; border-radius: 6px;
77
+ border: 1px solid #4a7c23; color: #2d5016; font-weight: 600;
78
+ font-size: 0.9em; transition: all 0.2s ease; }
79
+ .links a:hover { background: #4a7c23; color: #fff; }
80
+ .links a.primary { background: #2d5016; color: #fff; border-color: #2d5016; }
81
+ .links a.primary:hover { background: #4a7c23; border-color: #4a7c23; }
82
+ footer { margin-top: 48px; padding: 20px 0; border-top: 1px solid #f0ece0;
83
+ color: #6b6b5a; font-size: 0.85em; }
84
+ footer a { color: #2d5016; font-weight: 600; }
85
+ footer a:hover { color: #4a7c23; }
86
+ </style>
87
+ </head>
88
+ <body>
89
+ <div class="logo">
90
+ <img src="${LOGO_URL}" alt="vgi-rpc logo">
91
+ </div>
92
+ <h1>${escapeHtml(protocolName)}</h1>
93
+ <p class="meta">Powered by <code>vgi-rpc</code> (TypeScript) &middot; server <code>${escapeHtml(serverId)}</code></p>
94
+ <p>This is a <code>vgi-rpc</code> service endpoint.</p>
95
+ <div class="links">
96
+ ${links.join("\n")}
97
+ </div>
98
+ <footer>
99
+ &copy; 2026 &#x1F69C; <a href="https://query.farm">Query.Farm LLC</a>
100
+ </footer>
101
+ </body>
102
+ </html>`;
103
+ }
104
+
105
+ // ---------------------------------------------------------------------------
106
+ // 404 page
107
+ // ---------------------------------------------------------------------------
108
+
109
+ export function buildNotFoundPage(prefix: string, protocolName: string): string {
110
+ const nameFragment = protocolName ? ` (<strong>${escapeHtml(protocolName)}</strong>)` : "";
111
+ const prefixDisplay = prefix || "/";
112
+ return `<!DOCTYPE html>
113
+ <html lang="en">
114
+ <head>
115
+ <meta charset="utf-8">
116
+ <meta name="viewport" content="width=device-width, initial-scale=1">
117
+ <title>404 \u2014 vgi-rpc endpoint</title>
118
+ <style>
119
+ body { font-family: system-ui, -apple-system, sans-serif; max-width: 600px;
120
+ margin: 60px auto; padding: 0 20px; color: #333; text-align: center; }
121
+ .logo { margin-bottom: 24px; }
122
+ .logo img { width: 120px; height: 120px; }
123
+ h1 { color: #555; }
124
+ code { background: #f4f4f4; padding: 2px 6px; border-radius: 3px; font-size: 0.95em; }
125
+ a { color: #0066cc; }
126
+ p { line-height: 1.6; }
127
+ </style>
128
+ </head>
129
+ <body>
130
+ <div class="logo">
131
+ <img src="${LOGO_URL}" alt="vgi-rpc logo">
132
+ </div>
133
+ <h1>404 \u2014 Not Found</h1>
134
+ <p>This is a <code>vgi-rpc</code> service endpoint${nameFragment}.</p>
135
+ <p>RPC methods are available under <code>${escapeHtml(prefixDisplay)}/&lt;method&gt;</code>.</p>
136
+ <p>Learn more at <a href="https://vgi-rpc.query.farm">vgi-rpc.query.farm</a>.</p>
137
+ </body>
138
+ </html>`;
139
+ }
140
+
141
+ // ---------------------------------------------------------------------------
142
+ // Describe / API reference page
143
+ // ---------------------------------------------------------------------------
144
+
145
+ function buildMethodCard(method: MethodDefinition): string {
146
+ const name = escapeHtml(method.name);
147
+ const isUnary = method.type === "unary";
148
+ const hasHeader = !!method.headerSchema;
149
+
150
+ // Badges — match Python reference (unary/stream/header only)
151
+ const badges: string[] = [];
152
+ badges.push(
153
+ isUnary ? `<span class="badge badge-unary">unary</span>` : `<span class="badge badge-stream">stream</span>`,
154
+ );
155
+ if (hasHeader) badges.push(`<span class="badge badge-header">header</span>`);
156
+
157
+ // Parameters table
158
+ let paramsHtml = "";
159
+ const paramsSchema = method.paramsSchema;
160
+ if (paramsSchema.fields.length > 0) {
161
+ const rows = paramsSchema.fields.map((f) => {
162
+ const paramName = escapeHtml(f.name);
163
+ const paramType = escapeHtml(arrowTypeToString(f.type));
164
+ const defaultVal =
165
+ method.defaults && f.name in method.defaults ? escapeHtml(JSON.stringify(method.defaults[f.name])) : "&mdash;";
166
+ return `<tr><td><code>${paramName}</code></td><td><code>${paramType}</code></td><td>${defaultVal}</td><td>&mdash;</td></tr>`;
167
+ });
168
+ paramsHtml = `<div class="section-label">Parameters</div>
169
+ <table><tr><th>Name</th><th>Type</th><th>Default</th><th>Description</th></tr>
170
+ ${rows.join("\n")}
171
+ </table>`;
172
+ } else {
173
+ paramsHtml = `<p class="no-params">No parameters</p>`;
174
+ }
175
+
176
+ // Returns table (unary only)
177
+ let returnsHtml = "";
178
+ if (isUnary && method.resultSchema.fields.length > 0) {
179
+ const rows = method.resultSchema.fields.map((f) => {
180
+ return `<tr><td><code>${escapeHtml(f.name)}</code></td><td><code>${escapeHtml(arrowTypeToString(f.type))}</code></td></tr>`;
181
+ });
182
+ returnsHtml = `<div class="section-label">Returns</div>
183
+ <table><tr><th>Name</th><th>Type</th></tr>
184
+ ${rows.join("\n")}
185
+ </table>`;
186
+ }
187
+
188
+ // Header table (streams with headers)
189
+ let headerHtml = "";
190
+ if (hasHeader && method.headerSchema && method.headerSchema.fields.length > 0) {
191
+ const rows = method.headerSchema.fields.map((f) => {
192
+ return `<tr><td><code>${escapeHtml(f.name)}</code></td><td><code>${escapeHtml(arrowTypeToString(f.type))}</code></td></tr>`;
193
+ });
194
+ headerHtml = `<div class="section-label">Stream Header</div>
195
+ <table><tr><th>Name</th><th>Type</th></tr>
196
+ ${rows.join("\n")}
197
+ </table>`;
198
+ }
199
+
200
+ // Docstring
201
+ const docHtml = method.doc ? `<p class="docstring">${escapeHtml(method.doc)}</p>` : "";
202
+
203
+ return `<div class="card">
204
+ <div class="card-header">
205
+ <span class="method-name">${name}</span>
206
+ ${badges.join("\n")}
207
+ </div>
208
+ ${docHtml}
209
+ ${paramsHtml}
210
+ ${returnsHtml}
211
+ ${headerHtml}
212
+ </div>`;
213
+ }
214
+
215
+ export function buildDescribePage(
216
+ protocolName: string,
217
+ serverId: string,
218
+ methods: Map<string, MethodDefinition>,
219
+ repoUrl: string | null,
220
+ ): string {
221
+ const sortedMethods = [...methods.entries()]
222
+ .filter(([name]) => name !== "__describe__")
223
+ .sort(([a], [b]) => a.localeCompare(b));
224
+
225
+ const cards = sortedMethods.map(([, method]) => buildMethodCard(method)).join("\n");
226
+
227
+ const repoLink = repoUrl ? ` &middot; <a href="${escapeHtml(repoUrl)}">Source</a>` : "";
228
+
229
+ return `<!DOCTYPE html>
230
+ <html lang="en">
231
+ <head>
232
+ <meta charset="utf-8">
233
+ <meta name="viewport" content="width=device-width, initial-scale=1">
234
+ <title>${escapeHtml(protocolName)} API Reference \u2014 vgi-rpc</title>
235
+ ${FONTS}
236
+ <style>
237
+ body { font-family: 'Inter', system-ui, -apple-system, sans-serif; max-width: 900px;
238
+ margin: 0 auto; padding: 40px 20px 0; color: #2c2c1e; background: #faf8f0; }
239
+ .header { text-align: center; margin-bottom: 40px; }
240
+ .header .logo img { width: 80px; height: 80px; border-radius: 50%;
241
+ box-shadow: 0 3px 16px rgba(0,0,0,0.10); }
242
+ .header h1 { margin-bottom: 4px; color: #2d5016; font-weight: 700; }
243
+ .header .subtitle { color: #6b6b5a; font-size: 1.1em; margin-top: 0; }
244
+ .header .meta { color: #6b6b5a; font-size: 0.9em; }
245
+ .header .meta a { color: #2d5016; font-weight: 600; }
246
+ .header .meta a:hover { color: #4a7c23; }
247
+ code { font-family: 'JetBrains Mono', monospace; background: #f0ece0;
248
+ padding: 2px 6px; border-radius: 3px; font-size: 0.85em; color: #2c2c1e; }
249
+ a { color: #2d5016; text-decoration: none; }
250
+ a:hover { color: #4a7c23; }
251
+ .card { border: 1px solid #f0ece0; border-radius: 8px; padding: 20px;
252
+ margin-bottom: 16px; background: #fff; }
253
+ .card:hover { border-color: #c8a43a; }
254
+ .card-header { display: flex; align-items: center; gap: 10px; margin-bottom: 12px; }
255
+ .method-name { font-family: 'JetBrains Mono', monospace; font-size: 1.1em; font-weight: 600;
256
+ color: #2d5016; }
257
+ .badge { display: inline-block; padding: 2px 8px; border-radius: 4px;
258
+ font-size: 0.75em; font-weight: 600; text-transform: uppercase;
259
+ letter-spacing: 0.03em; }
260
+ .badge-unary { background: #e8f5e0; color: #2d5016; }
261
+ .badge-stream { background: #e0ecf5; color: #1a4a6b; }
262
+ .badge-exchange { background: #f5e6f0; color: #6b234a; }
263
+ .badge-producer { background: #e0f0f5; color: #1a5a6b; }
264
+ .badge-header { background: #f5eee0; color: #6b4423; }
265
+ .docstring { color: #6b6b5a; font-size: 0.9em; margin-top: 0; }
266
+ table { width: 100%; border-collapse: collapse; font-size: 0.9em; }
267
+ th { text-align: left; padding: 8px 10px; background: #f0ece0; color: #2c2c1e;
268
+ font-weight: 600; border-bottom: 2px solid #e0dcd0; }
269
+ td { padding: 8px 10px; border-bottom: 1px solid #f0ece0; }
270
+ td code { font-size: 0.85em; }
271
+ .no-params { color: #6b6b5a; font-style: italic; font-size: 0.9em; }
272
+ .section-label { font-size: 0.8em; font-weight: 600; text-transform: uppercase;
273
+ letter-spacing: 0.05em; color: #6b6b5a; margin-top: 14px;
274
+ margin-bottom: 6px; }
275
+ footer { text-align: center; margin-top: 48px; padding: 20px 0;
276
+ border-top: 1px solid #f0ece0; color: #6b6b5a; font-size: 0.85em; }
277
+ footer a { color: #2d5016; font-weight: 600; }
278
+ footer a:hover { color: #4a7c23; }
279
+ </style>
280
+ </head>
281
+ <body>
282
+ <div class="header">
283
+ <div class="logo">
284
+ <img src="${LOGO_URL}" alt="vgi-rpc logo">
285
+ </div>
286
+ <h1>${escapeHtml(protocolName)}</h1>
287
+ <p class="subtitle">API Reference</p>
288
+ <p class="meta">Powered by <code>vgi-rpc</code> (TypeScript) &middot; server <code>${escapeHtml(serverId)}</code>${repoLink}</p>
289
+ </div>
290
+ ${cards}
291
+ <footer>
292
+ <a href="https://vgi-rpc.query.farm">Learn more about <code>vgi-rpc</code></a>
293
+ &middot;
294
+ &copy; 2026 &#x1F69C; <a href="https://query.farm">Query.Farm LLC</a>
295
+ </footer>
296
+ </body>
297
+ </html>`;
298
+ }
package/src/http/types.ts CHANGED
@@ -1,11 +1,13 @@
1
1
  // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
 
4
+ import type { ExternalLocationConfig } from "../external.js";
5
+ import type { DispatchHook } from "../types.js";
4
6
  import type { AuthenticateFn, OAuthResourceMetadata } from "./auth.js";
5
7
 
6
8
  /** Configuration options for createHttpHandler(). */
7
9
  export interface HttpHandlerOptions {
8
- /** URL path prefix for all endpoints. Default: "/vgi" */
10
+ /** URL path prefix for all endpoints. Default: "" (root). */
9
11
  prefix?: string;
10
12
  /** HMAC-SHA256 signing key for state tokens. Random 32 bytes if omitted. */
11
13
  signingKey?: Uint8Array;
@@ -28,6 +30,20 @@ export interface HttpHandlerOptions {
28
30
  authenticate?: AuthenticateFn;
29
31
  /** Optional RFC 9728 OAuth Protected Resource Metadata. Served at well-known endpoint. */
30
32
  oauthResourceMetadata?: OAuthResourceMetadata;
33
+ /** Optional dispatch hook for observability (tracing, metrics). */
34
+ dispatchHook?: DispatchHook;
35
+ /** Enable HTML landing page at GET {prefix}/. Default: true. */
36
+ enableLandingPage?: boolean;
37
+ /** Enable HTML describe/API reference page at GET {prefix}/describe. Default: true. */
38
+ enableDescribePage?: boolean;
39
+ /** Enable HTML 404 page for unmatched GET routes. Default: true. */
40
+ enableNotFoundPage?: boolean;
41
+ /** Protocol name shown in HTML pages. Defaults to the Protocol's name. */
42
+ protocolName?: string;
43
+ /** URL to service's source repository, shown in landing/describe pages. */
44
+ repositoryUrl?: string;
45
+ /** External storage config for externalizing large response batches. */
46
+ externalLocation?: ExternalLocationConfig;
31
47
  }
32
48
 
33
49
  /** Serializer for stream state objects stored in state tokens. */
package/src/index.ts CHANGED
@@ -19,19 +19,40 @@ export {
19
19
  STATE_KEY,
20
20
  } from "./constants.js";
21
21
  export { RpcError, VersionError } from "./errors.js";
22
+ export {
23
+ type ExternalLocationConfig,
24
+ type ExternalStorage,
25
+ httpsOnlyValidator,
26
+ isExternalLocationBatch,
27
+ makeExternalLocationBatch,
28
+ maybeExternalizeBatch,
29
+ resolveExternalLocation,
30
+ } from "./external.js";
22
31
  export {
23
32
  ARROW_CONTENT_TYPE,
24
33
  type AuthenticateFn,
34
+ type BearerValidateFn,
35
+ bearerAuthenticate,
36
+ bearerAuthenticateStatic,
37
+ type CertValidateFn,
38
+ chainAuthenticate,
25
39
  createHttpHandler,
26
40
  type HttpHandlerOptions,
27
41
  type JwtAuthenticateOptions,
28
42
  jsonStateSerializer,
29
43
  jwtAuthenticate,
44
+ mtlsAuthenticate,
45
+ mtlsAuthenticateFingerprint,
46
+ mtlsAuthenticateSubject,
47
+ mtlsAuthenticateXfcc,
30
48
  type OAuthResourceMetadata,
31
49
  oauthResourceMetadataToJson,
50
+ parseXfcc,
32
51
  type StateSerializer,
33
52
  type UnpackedToken,
34
53
  unpackStateToken,
54
+ type XfccElement,
55
+ type XfccValidateFn,
35
56
  } from "./http/index.js";
36
57
  export { Protocol } from "./protocol.js";
37
58
  export {
@@ -49,9 +70,13 @@ export {
49
70
  export { VgiRpcServer } from "./server.js";
50
71
  export {
51
72
  type CallContext,
73
+ type CallStatistics,
74
+ type DispatchHook,
75
+ type DispatchInfo,
52
76
  type ExchangeFn,
53
77
  type ExchangeInit,
54
78
  type HeaderInit,
79
+ type HookToken,
55
80
  type LogContext,
56
81
  type MethodDefinition,
57
82
  MethodType,
package/src/otel.ts ADDED
@@ -0,0 +1,161 @@
1
+ // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
+ // SPDX-License-Identifier: Apache-2.0
3
+
4
+ /**
5
+ * OpenTelemetry instrumentation for vgi-rpc TypeScript servers.
6
+ *
7
+ * Implements {@link DispatchHook} to add distributed tracing (spans) and
8
+ * metrics (request counter, duration histogram) to RPC dispatch.
9
+ *
10
+ * Requires `@opentelemetry/api` as a peer dependency.
11
+ *
12
+ * @example
13
+ * ```typescript
14
+ * import { createOtelHook } from "vgi-rpc/otel";
15
+ * import { createHttpHandler } from "vgi-rpc";
16
+ *
17
+ * const handler = createHttpHandler(protocol, {
18
+ * dispatchHook: createOtelHook(),
19
+ * });
20
+ * ```
21
+ */
22
+
23
+ import {
24
+ type Attributes,
25
+ type Counter,
26
+ type Histogram,
27
+ type Meter,
28
+ metrics,
29
+ type Span,
30
+ SpanKind,
31
+ SpanStatusCode,
32
+ type Tracer,
33
+ trace,
34
+ } from "@opentelemetry/api";
35
+ import type { CallStatistics, DispatchHook, DispatchInfo, HookToken } from "./types.js";
36
+
37
+ const INSTRUMENTATION_NAME = "vgi_rpc";
38
+
39
+ /** Configuration for OpenTelemetry instrumentation. */
40
+ export interface OtelConfig {
41
+ /** Custom TracerProvider; uses the global provider when omitted. */
42
+ tracerProvider?: { getTracer(name: string): Tracer };
43
+ /** Custom MeterProvider; uses the global provider when omitted. */
44
+ meterProvider?: { getMeter(name: string): Meter };
45
+ /** Enable span creation. Default: true. */
46
+ enableTracing?: boolean;
47
+ /** Enable counter/histogram recording. Default: true. */
48
+ enableMetrics?: boolean;
49
+ /** Record exceptions on error spans. Default: true. */
50
+ recordExceptions?: boolean;
51
+ /** Service name for the rpc.service attribute. Default: "TypeScriptRpcServer". */
52
+ serviceName?: string;
53
+ }
54
+
55
+ interface OtelHookToken {
56
+ span: Span | null;
57
+ startTime: number;
58
+ }
59
+
60
+ /**
61
+ * Create a {@link DispatchHook} that instruments RPC calls with OpenTelemetry.
62
+ *
63
+ * Creates a span for each RPC call with method attributes, and records
64
+ * request count and duration metrics.
65
+ */
66
+ export function createOtelHook(config?: OtelConfig): DispatchHook {
67
+ const enableTracing = config?.enableTracing ?? true;
68
+ const enableMetrics = config?.enableMetrics ?? true;
69
+ const recordExceptions = config?.recordExceptions ?? true;
70
+ const serviceName = config?.serviceName ?? "TypeScriptRpcServer";
71
+
72
+ const tracer = (config?.tracerProvider ?? trace).getTracer(INSTRUMENTATION_NAME);
73
+
74
+ let requestCounter: Counter | null = null;
75
+ let durationHistogram: Histogram | null = null;
76
+
77
+ if (enableMetrics) {
78
+ const meter = (config?.meterProvider ?? metrics).getMeter(INSTRUMENTATION_NAME);
79
+ requestCounter = meter.createCounter("rpc.server.requests", {
80
+ unit: "{request}",
81
+ description: "Number of RPC requests handled",
82
+ });
83
+ durationHistogram = meter.createHistogram("rpc.server.duration", {
84
+ unit: "s",
85
+ description: "Duration of RPC requests",
86
+ });
87
+ }
88
+
89
+ return {
90
+ onDispatchStart(info: DispatchInfo): HookToken {
91
+ const startTime = performance.now();
92
+
93
+ if (!enableTracing) {
94
+ return { span: null, startTime } satisfies OtelHookToken;
95
+ }
96
+
97
+ const spanName = `vgi_rpc/${info.method}`;
98
+ const attrs: Attributes = {
99
+ "rpc.system": "vgi_rpc",
100
+ "rpc.service": serviceName,
101
+ "rpc.method": info.method,
102
+ "rpc.vgi_rpc.method_type": info.methodType,
103
+ "rpc.vgi_rpc.server_id": info.serverId,
104
+ };
105
+ if (info.requestId) {
106
+ attrs["rpc.vgi_rpc.request_id"] = info.requestId;
107
+ }
108
+
109
+ const span = tracer.startSpan(spanName, {
110
+ kind: SpanKind.SERVER,
111
+ attributes: attrs,
112
+ });
113
+
114
+ return { span, startTime } satisfies OtelHookToken;
115
+ },
116
+
117
+ onDispatchEnd(token: HookToken, info: DispatchInfo, stats: CallStatistics, error?: Error): void {
118
+ const t = token as OtelHookToken;
119
+ const durationS = (performance.now() - t.startTime) / 1000;
120
+ const status = error ? "error" : "ok";
121
+
122
+ // Finalize span
123
+ if (t.span) {
124
+ if (stats) {
125
+ t.span.setAttributes({
126
+ "rpc.vgi_rpc.input_batches": stats.inputBatches,
127
+ "rpc.vgi_rpc.output_batches": stats.outputBatches,
128
+ "rpc.vgi_rpc.input_rows": stats.inputRows,
129
+ "rpc.vgi_rpc.output_rows": stats.outputRows,
130
+ "rpc.vgi_rpc.input_bytes": stats.inputBytes,
131
+ "rpc.vgi_rpc.output_bytes": stats.outputBytes,
132
+ });
133
+ }
134
+
135
+ if (error) {
136
+ t.span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
137
+ t.span.setAttribute("rpc.vgi_rpc.error_type", error.constructor.name);
138
+ if (recordExceptions) {
139
+ t.span.recordException(error);
140
+ }
141
+ } else {
142
+ t.span.setStatus({ code: SpanStatusCode.OK });
143
+ }
144
+ t.span.end();
145
+ }
146
+
147
+ // Record metrics
148
+ if (enableMetrics) {
149
+ const metricAttrs: Attributes = {
150
+ "rpc.system": "vgi_rpc",
151
+ "rpc.service": serviceName,
152
+ "rpc.method": info.method,
153
+ "rpc.vgi_rpc.method_type": info.methodType,
154
+ status,
155
+ };
156
+ requestCounter?.add(1, metricAttrs);
157
+ durationHistogram?.record(durationS, metricAttrs);
158
+ }
159
+ },
160
+ };
161
+ }
package/src/s3.ts ADDED
@@ -0,0 +1,94 @@
1
+ // © Copyright 2025-2026, Query.Farm LLC - https://query.farm
2
+ // SPDX-License-Identifier: Apache-2.0
3
+
4
+ /**
5
+ * S3 storage backend for external storage of large Arrow IPC batches.
6
+ *
7
+ * Requires `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
8
+ * as peer dependencies.
9
+ *
10
+ * @example
11
+ * ```typescript
12
+ * import { createS3Storage } from "@query-farm/vgi-rpc/s3";
13
+ *
14
+ * const storage = createS3Storage({
15
+ * bucket: "my-bucket",
16
+ * prefix: "vgi-rpc/",
17
+ * });
18
+ * const handler = createHttpHandler(protocol, {
19
+ * externalLocation: { storage, externalizeThresholdBytes: 1_048_576 },
20
+ * });
21
+ * ```
22
+ */
23
+
24
+ import type { ExternalStorage } from "./external.js";
25
+
26
+ /** Configuration for the S3 storage backend. */
27
+ export interface S3StorageConfig {
28
+ /** S3 bucket name. */
29
+ bucket: string;
30
+ /** Key prefix for uploaded objects. Default: "vgi-rpc/". */
31
+ prefix?: string;
32
+ /** Lifetime of pre-signed GET URLs in seconds. Default: 3600 (1 hour). */
33
+ presignExpirySeconds?: number;
34
+ /** AWS region. If omitted, uses default SDK config. */
35
+ region?: string;
36
+ /** Custom S3 endpoint URL (for MinIO, LocalStack, etc.). */
37
+ endpointUrl?: string;
38
+ /** Force path-style addressing (required for some S3-compatible services). */
39
+ forcePathStyle?: boolean;
40
+ }
41
+
42
+ /**
43
+ * Create an S3-backed ExternalStorage.
44
+ *
45
+ * Lazily imports `@aws-sdk/client-s3` and `@aws-sdk/s3-request-presigner`
46
+ * on first upload to avoid loading the AWS SDK unless needed.
47
+ */
48
+ export function createS3Storage(config: S3StorageConfig): ExternalStorage {
49
+ const bucket = config.bucket;
50
+ const prefix = config.prefix ?? "vgi-rpc/";
51
+ const presignExpiry = config.presignExpirySeconds ?? 3600;
52
+
53
+ // Lazy-loaded AWS SDK clients
54
+ let s3Client: any = null;
55
+
56
+ async function ensureClient(): Promise<any> {
57
+ if (s3Client) return s3Client;
58
+ const { S3Client } = await import("@aws-sdk/client-s3");
59
+ const clientConfig: Record<string, any> = {};
60
+ if (config.region) clientConfig.region = config.region;
61
+ if (config.endpointUrl) {
62
+ clientConfig.endpoint = config.endpointUrl;
63
+ clientConfig.forcePathStyle = config.forcePathStyle ?? true;
64
+ }
65
+ s3Client = new S3Client(clientConfig);
66
+ return s3Client;
67
+ }
68
+
69
+ return {
70
+ async upload(data: Uint8Array, contentEncoding: string): Promise<string> {
71
+ const client = await ensureClient();
72
+ const { PutObjectCommand } = await import("@aws-sdk/client-s3");
73
+ const { getSignedUrl } = await import("@aws-sdk/s3-request-presigner");
74
+ const { GetObjectCommand } = await import("@aws-sdk/client-s3");
75
+
76
+ const key = `${prefix}${crypto.randomUUID()}${contentEncoding === "zstd" ? ".arrow.zst" : ".arrow"}`;
77
+
78
+ const putCommand = new PutObjectCommand({
79
+ Bucket: bucket,
80
+ Key: key,
81
+ Body: data,
82
+ ContentType: "application/vnd.apache.arrow.stream",
83
+ ...(contentEncoding ? { ContentEncoding: contentEncoding } : {}),
84
+ });
85
+
86
+ await client.send(putCommand);
87
+
88
+ // Generate pre-signed GET URL
89
+ const getCommand = new GetObjectCommand({ Bucket: bucket, Key: key });
90
+ const url = await getSignedUrl(client, getCommand, { expiresIn: presignExpiry });
91
+ return url;
92
+ },
93
+ };
94
+ }