@query-farm/vgi-rpc 0.7.5 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/client/index.d.ts +1 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/tcp.d.ts +19 -0
- package/dist/client/tcp.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1519 -1260
- package/dist/index.js.map +12 -10
- package/dist/launcher/index.d.ts +1 -0
- package/dist/launcher/index.d.ts.map +1 -1
- package/dist/launcher/serve-tcp.d.ts +66 -0
- package/dist/launcher/serve-tcp.d.ts.map +1 -0
- package/dist/types.d.ts +6 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client/index.ts +1 -0
- package/src/client/tcp.ts +68 -0
- package/src/client/types.ts +8 -0
- package/src/index.ts +6 -0
- package/src/launcher/index.ts +1 -0
- package/src/launcher/serve-tcp.ts +382 -0
- package/src/types.ts +5 -0
package/dist/index.js
CHANGED
|
@@ -734,130 +734,42 @@ var init_zstd = __esm(() => {
|
|
|
734
734
|
isBun = typeof globalThis.Bun !== "undefined";
|
|
735
735
|
});
|
|
736
736
|
|
|
737
|
-
// src/
|
|
738
|
-
|
|
739
|
-
function _loadWriteSync() {
|
|
740
|
-
const req = import.meta.require ?? globalThis.require ?? null;
|
|
741
|
-
if (!req) {
|
|
742
|
-
throw new Error("FdSink requires Node.js or Bun (node:fs.writeSync). For other runtimes, " + "supply a custom AccessLogSink that wraps console.log or your logger.");
|
|
743
|
-
}
|
|
744
|
-
return req(_NODE_FS_MOD).writeSync;
|
|
745
|
-
}
|
|
737
|
+
// src/client/tcp.ts
|
|
738
|
+
import { connect } from "node:net";
|
|
746
739
|
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
return Math.round(f * 100) / 100;
|
|
780
|
-
}
|
|
740
|
+
// src/client/pipe.ts
|
|
741
|
+
import {
|
|
742
|
+
Field,
|
|
743
|
+
makeData,
|
|
744
|
+
RecordBatch,
|
|
745
|
+
RecordBatchStreamWriter as RecordBatchStreamWriter2,
|
|
746
|
+
Schema,
|
|
747
|
+
Struct,
|
|
748
|
+
vectorFromArray
|
|
749
|
+
} from "@query-farm/apache-arrow";
|
|
750
|
+
|
|
751
|
+
// src/constants.ts
|
|
752
|
+
var RPC_METHOD_KEY = "vgi_rpc.method";
|
|
753
|
+
var LOG_LEVEL_KEY = "vgi_rpc.log_level";
|
|
754
|
+
var LOG_MESSAGE_KEY = "vgi_rpc.log_message";
|
|
755
|
+
var LOG_EXTRA_KEY = "vgi_rpc.log_extra";
|
|
756
|
+
var REQUEST_VERSION_KEY = "vgi_rpc.request_version";
|
|
757
|
+
var REQUEST_VERSION = "1";
|
|
758
|
+
var SERVER_ID_KEY = "vgi_rpc.server_id";
|
|
759
|
+
var REQUEST_ID_KEY = "vgi_rpc.request_id";
|
|
760
|
+
var PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
|
|
761
|
+
var DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
|
|
762
|
+
var PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
|
|
763
|
+
var DESCRIBE_VERSION = "4";
|
|
764
|
+
var PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
|
|
765
|
+
var DESCRIBE_METHOD_NAME = "__describe__";
|
|
766
|
+
var STATE_KEY = "vgi_rpc.stream_state#b64";
|
|
767
|
+
var CANCEL_KEY = "vgi_rpc.cancel";
|
|
768
|
+
var LOCATION_KEY = "vgi_rpc.location";
|
|
769
|
+
var LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
|
|
770
|
+
var RPC_ERROR_HEADER = "X-VGI-RPC-Error";
|
|
771
|
+
var ERROR_KIND_KEY = "vgi_rpc.error_kind";
|
|
781
772
|
|
|
782
|
-
class AccessLogHook {
|
|
783
|
-
sink;
|
|
784
|
-
serverVersion;
|
|
785
|
-
level;
|
|
786
|
-
constructor(sink, options = {}) {
|
|
787
|
-
this.sink = sink;
|
|
788
|
-
if (typeof options === "string") {
|
|
789
|
-
this.serverVersion = options;
|
|
790
|
-
this.level = "INFO";
|
|
791
|
-
} else {
|
|
792
|
-
this.serverVersion = options.serverVersion ?? "";
|
|
793
|
-
this.level = options.level ?? "INFO";
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
onDispatchStart(_info) {
|
|
797
|
-
const token = { startNs: process.hrtime.bigint() };
|
|
798
|
-
return token;
|
|
799
|
-
}
|
|
800
|
-
onDispatchEnd(token, info, stats, error) {
|
|
801
|
-
const t = token;
|
|
802
|
-
const durationMs = t ? roundTo2(Number(process.hrtime.bigint() - t.startNs) / 1e6) : 0;
|
|
803
|
-
const status = error ? "error" : "ok";
|
|
804
|
-
const errType = error ? error.type ?? error.constructor.name : "";
|
|
805
|
-
const errMsg = error?.message ?? "";
|
|
806
|
-
const protocol = info.protocol ?? "";
|
|
807
|
-
const rec = {
|
|
808
|
-
timestamp: rfc3339Utc(),
|
|
809
|
-
level: "INFO",
|
|
810
|
-
logger: "vgi_rpc.access",
|
|
811
|
-
message: `${protocol}.${info.method} ${status}`,
|
|
812
|
-
server_id: info.serverId,
|
|
813
|
-
protocol,
|
|
814
|
-
protocol_hash: info.protocolHash ?? "",
|
|
815
|
-
method: info.method,
|
|
816
|
-
method_type: info.methodType,
|
|
817
|
-
principal: info.principal ?? "",
|
|
818
|
-
auth_domain: info.authDomain ?? "",
|
|
819
|
-
authenticated: info.authenticated ?? false,
|
|
820
|
-
remote_addr: info.remoteAddr ?? "",
|
|
821
|
-
duration_ms: durationMs,
|
|
822
|
-
status,
|
|
823
|
-
error_type: errType
|
|
824
|
-
};
|
|
825
|
-
if (errMsg)
|
|
826
|
-
rec.error_message = errMsg;
|
|
827
|
-
if (this.serverVersion)
|
|
828
|
-
rec.server_version = this.serverVersion;
|
|
829
|
-
if (info.protocolVersion)
|
|
830
|
-
rec.protocol_version = info.protocolVersion;
|
|
831
|
-
if (info.requestId)
|
|
832
|
-
rec.request_id = info.requestId;
|
|
833
|
-
if (info.requestData && info.requestData.length > 0) {
|
|
834
|
-
const encoded = base64(info.requestData);
|
|
835
|
-
if (this.level === "DEBUG") {
|
|
836
|
-
rec.request_data = encoded;
|
|
837
|
-
} else {
|
|
838
|
-
rec.original_request_bytes = encoded.length;
|
|
839
|
-
rec.truncated = true;
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
if (info.methodType === "stream") {
|
|
843
|
-
rec.stream_id = info.streamId ?? "00000000000000000000000000000000";
|
|
844
|
-
}
|
|
845
|
-
if (info.cancelled)
|
|
846
|
-
rec.cancelled = true;
|
|
847
|
-
if (stats.inputBatches + stats.outputBatches + stats.inputRows + stats.outputRows + stats.inputBytes + stats.outputBytes !== 0) {
|
|
848
|
-
rec.input_batches = stats.inputBatches;
|
|
849
|
-
rec.output_batches = stats.outputBatches;
|
|
850
|
-
rec.input_rows = stats.inputRows;
|
|
851
|
-
rec.output_rows = stats.outputRows;
|
|
852
|
-
rec.input_bytes = stats.inputBytes;
|
|
853
|
-
rec.output_bytes = stats.outputBytes;
|
|
854
|
-
}
|
|
855
|
-
try {
|
|
856
|
-
this.sink.write(`${JSON.stringify(rec)}
|
|
857
|
-
`);
|
|
858
|
-
} catch {}
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
773
|
// src/errors.ts
|
|
862
774
|
class RpcError extends Error {
|
|
863
775
|
errorType;
|
|
@@ -927,49 +839,6 @@ class ServerDrainingError extends Error {
|
|
|
927
839
|
}
|
|
928
840
|
}
|
|
929
841
|
|
|
930
|
-
// src/auth.ts
|
|
931
|
-
class AuthContext {
|
|
932
|
-
domain;
|
|
933
|
-
authenticated;
|
|
934
|
-
principal;
|
|
935
|
-
claims;
|
|
936
|
-
constructor(domain, authenticated, principal, claims = {}) {
|
|
937
|
-
this.domain = domain;
|
|
938
|
-
this.authenticated = authenticated;
|
|
939
|
-
this.principal = principal;
|
|
940
|
-
this.claims = claims;
|
|
941
|
-
}
|
|
942
|
-
static anonymous() {
|
|
943
|
-
return new AuthContext("", false, null);
|
|
944
|
-
}
|
|
945
|
-
requireAuthenticated() {
|
|
946
|
-
if (!this.authenticated) {
|
|
947
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
// src/constants.ts
|
|
952
|
-
var RPC_METHOD_KEY = "vgi_rpc.method";
|
|
953
|
-
var LOG_LEVEL_KEY = "vgi_rpc.log_level";
|
|
954
|
-
var LOG_MESSAGE_KEY = "vgi_rpc.log_message";
|
|
955
|
-
var LOG_EXTRA_KEY = "vgi_rpc.log_extra";
|
|
956
|
-
var REQUEST_VERSION_KEY = "vgi_rpc.request_version";
|
|
957
|
-
var REQUEST_VERSION = "1";
|
|
958
|
-
var SERVER_ID_KEY = "vgi_rpc.server_id";
|
|
959
|
-
var REQUEST_ID_KEY = "vgi_rpc.request_id";
|
|
960
|
-
var PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
|
|
961
|
-
var DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
|
|
962
|
-
var PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
|
|
963
|
-
var DESCRIBE_VERSION = "4";
|
|
964
|
-
var PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
|
|
965
|
-
var DESCRIBE_METHOD_NAME = "__describe__";
|
|
966
|
-
var STATE_KEY = "vgi_rpc.stream_state#b64";
|
|
967
|
-
var CANCEL_KEY = "vgi_rpc.cancel";
|
|
968
|
-
var LOCATION_KEY = "vgi_rpc.location";
|
|
969
|
-
var LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
|
|
970
|
-
var RPC_ERROR_HEADER = "X-VGI-RPC-Error";
|
|
971
|
-
var ERROR_KIND_KEY = "vgi_rpc.error_kind";
|
|
972
|
-
|
|
973
842
|
// src/arrow/impl-arrowjs/index.ts
|
|
974
843
|
import {
|
|
975
844
|
Binary as A_Binary,
|
|
@@ -1509,63 +1378,6 @@ async function readRequestFromBody(body) {
|
|
|
1509
1378
|
return { schema: batch.schema, batch };
|
|
1510
1379
|
}
|
|
1511
1380
|
|
|
1512
|
-
// src/client/capabilities.ts
|
|
1513
|
-
var MAX_REQUEST_BYTES_HEADER = "VGI-Max-Request-Bytes";
|
|
1514
|
-
var UPLOAD_URL_HEADER = "VGI-Upload-URL-Support";
|
|
1515
|
-
var MAX_UPLOAD_BYTES_HEADER = "VGI-Max-Upload-Bytes";
|
|
1516
|
-
function parseHeaderInt(headers, name) {
|
|
1517
|
-
const raw = headers.get(name) ?? headers.get(name.toLowerCase());
|
|
1518
|
-
if (raw == null)
|
|
1519
|
-
return null;
|
|
1520
|
-
const parsed = Number.parseInt(raw, 10);
|
|
1521
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
1522
|
-
}
|
|
1523
|
-
function parseCapabilitiesFromHeaders(headers) {
|
|
1524
|
-
const uploadRaw = headers.get(UPLOAD_URL_HEADER) ?? headers.get(UPLOAD_URL_HEADER.toLowerCase());
|
|
1525
|
-
const uploadUrlSupport = uploadRaw === "true";
|
|
1526
|
-
let cacheExpiresAt = null;
|
|
1527
|
-
const cc = headers.get("Cache-Control") ?? headers.get("cache-control");
|
|
1528
|
-
if (cc) {
|
|
1529
|
-
for (const token of cc.split(",")) {
|
|
1530
|
-
const t = token.trim().toLowerCase();
|
|
1531
|
-
if (t.startsWith("max-age=")) {
|
|
1532
|
-
const seconds = Number.parseFloat(t.slice("max-age=".length));
|
|
1533
|
-
if (Number.isFinite(seconds)) {
|
|
1534
|
-
cacheExpiresAt = Date.now() + seconds * 1000;
|
|
1535
|
-
}
|
|
1536
|
-
break;
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
}
|
|
1540
|
-
return {
|
|
1541
|
-
maxRequestBytes: parseHeaderInt(headers, MAX_REQUEST_BYTES_HEADER),
|
|
1542
|
-
uploadUrlSupport,
|
|
1543
|
-
maxUploadBytes: parseHeaderInt(headers, MAX_UPLOAD_BYTES_HEADER),
|
|
1544
|
-
cacheExpiresAt
|
|
1545
|
-
};
|
|
1546
|
-
}
|
|
1547
|
-
function isCapabilitySnapshotFresh(snapshot) {
|
|
1548
|
-
if (!snapshot)
|
|
1549
|
-
return false;
|
|
1550
|
-
if (snapshot.cacheExpiresAt == null)
|
|
1551
|
-
return true;
|
|
1552
|
-
return Date.now() < snapshot.cacheExpiresAt;
|
|
1553
|
-
}
|
|
1554
|
-
|
|
1555
|
-
// src/client/introspect.ts
|
|
1556
|
-
import { Schema as ArrowSchema } from "@query-farm/apache-arrow";
|
|
1557
|
-
|
|
1558
|
-
// src/client/ipc.ts
|
|
1559
|
-
import {
|
|
1560
|
-
Binary,
|
|
1561
|
-
Bool,
|
|
1562
|
-
DataType,
|
|
1563
|
-
Float64,
|
|
1564
|
-
Int64,
|
|
1565
|
-
RecordBatchReader as RecordBatchReader3,
|
|
1566
|
-
Utf8
|
|
1567
|
-
} from "@query-farm/apache-arrow";
|
|
1568
|
-
|
|
1569
1381
|
// src/wire/reader.ts
|
|
1570
1382
|
import { RecordBatchReader as RecordBatchReader2 } from "@query-farm/apache-arrow";
|
|
1571
1383
|
|
|
@@ -1637,12 +1449,24 @@ class IpcStreamReader {
|
|
|
1637
1449
|
}
|
|
1638
1450
|
}
|
|
1639
1451
|
|
|
1452
|
+
// src/client/introspect.ts
|
|
1453
|
+
import { Schema as ArrowSchema } from "@query-farm/apache-arrow";
|
|
1454
|
+
|
|
1640
1455
|
// src/client/ipc.ts
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1456
|
+
import {
|
|
1457
|
+
Binary,
|
|
1458
|
+
Bool,
|
|
1459
|
+
DataType,
|
|
1460
|
+
Float64,
|
|
1461
|
+
Int64,
|
|
1462
|
+
RecordBatchReader as RecordBatchReader3,
|
|
1463
|
+
Utf8
|
|
1464
|
+
} from "@query-farm/apache-arrow";
|
|
1465
|
+
function inferArrowType(value) {
|
|
1466
|
+
if (typeof value === "string")
|
|
1467
|
+
return new Utf8;
|
|
1468
|
+
if (typeof value === "boolean")
|
|
1469
|
+
return new Bool;
|
|
1646
1470
|
if (typeof value === "bigint")
|
|
1647
1471
|
return new Int64;
|
|
1648
1472
|
if (typeof value === "number")
|
|
@@ -1853,1173 +1677,1380 @@ async function httpIntrospect(baseUrl, options) {
|
|
|
1853
1677
|
return parseDescribeResponse(batches);
|
|
1854
1678
|
}
|
|
1855
1679
|
|
|
1856
|
-
// src/client/
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1680
|
+
// src/client/pipe.ts
|
|
1681
|
+
class PipeIncrementalWriter {
|
|
1682
|
+
writer;
|
|
1683
|
+
writeFn;
|
|
1684
|
+
closed = false;
|
|
1685
|
+
constructor(writeFn, schema2) {
|
|
1686
|
+
this.writeFn = writeFn;
|
|
1687
|
+
this.writer = new RecordBatchStreamWriter2;
|
|
1688
|
+
this.writer.reset(undefined, schema2);
|
|
1689
|
+
this.drain();
|
|
1690
|
+
}
|
|
1691
|
+
write(batch) {
|
|
1692
|
+
if (this.closed)
|
|
1693
|
+
throw new Error("PipeIncrementalWriter already closed");
|
|
1694
|
+
this.writer._writeRecordBatch(batch);
|
|
1695
|
+
this.drain();
|
|
1696
|
+
}
|
|
1697
|
+
close() {
|
|
1698
|
+
if (this.closed)
|
|
1699
|
+
return;
|
|
1700
|
+
this.closed = true;
|
|
1701
|
+
const eos = new Uint8Array(new Int32Array([-1, 0]).buffer);
|
|
1702
|
+
this.writeFn(eos);
|
|
1703
|
+
}
|
|
1704
|
+
drain() {
|
|
1705
|
+
const values = this.writer._sink._values;
|
|
1706
|
+
for (const chunk of values) {
|
|
1707
|
+
this.writeFn(chunk);
|
|
1708
|
+
}
|
|
1709
|
+
values.length = 0;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
class PipeStreamSession {
|
|
1714
|
+
_reader;
|
|
1715
|
+
_writeFn;
|
|
1865
1716
|
_onLog;
|
|
1866
|
-
_pendingBatches;
|
|
1867
|
-
_finished;
|
|
1868
1717
|
_header;
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1718
|
+
_inputWriter = null;
|
|
1719
|
+
_inputSchema = null;
|
|
1720
|
+
_outputStreamOpened = false;
|
|
1721
|
+
_closed = false;
|
|
1722
|
+
_outputSchema;
|
|
1723
|
+
_releaseBusy;
|
|
1724
|
+
_setDrainPromise;
|
|
1873
1725
|
_externalConfig;
|
|
1874
|
-
_postFn;
|
|
1875
1726
|
constructor(opts) {
|
|
1876
|
-
this.
|
|
1877
|
-
this.
|
|
1878
|
-
this._method = opts.method;
|
|
1879
|
-
this._stateToken = opts.stateToken;
|
|
1880
|
-
this._outputSchema = opts.outputSchema;
|
|
1881
|
-
this._inputSchema = opts.inputSchema;
|
|
1727
|
+
this._reader = opts.reader;
|
|
1728
|
+
this._writeFn = opts.writeFn;
|
|
1882
1729
|
this._onLog = opts.onLog;
|
|
1883
|
-
this._pendingBatches = opts.pendingBatches;
|
|
1884
|
-
this._finished = opts.finished;
|
|
1885
1730
|
this._header = opts.header;
|
|
1886
|
-
this.
|
|
1887
|
-
this.
|
|
1888
|
-
this.
|
|
1889
|
-
this._authorization = opts.authorization;
|
|
1731
|
+
this._outputSchema = opts.outputSchema;
|
|
1732
|
+
this._releaseBusy = opts.releaseBusy;
|
|
1733
|
+
this._setDrainPromise = opts.setDrainPromise;
|
|
1890
1734
|
this._externalConfig = opts.externalConfig;
|
|
1891
|
-
this._postFn = opts.postFn;
|
|
1892
|
-
}
|
|
1893
|
-
async _post(url, body) {
|
|
1894
|
-
if (this._postFn)
|
|
1895
|
-
return this._postFn(url, body);
|
|
1896
|
-
return fetch(url, {
|
|
1897
|
-
method: "POST",
|
|
1898
|
-
headers: this._buildHeaders(),
|
|
1899
|
-
body: await this._prepareBody(body)
|
|
1900
|
-
});
|
|
1901
1735
|
}
|
|
1902
1736
|
get header() {
|
|
1903
1737
|
return this._header;
|
|
1904
1738
|
}
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
}
|
|
1920
|
-
async _prepareBody(content) {
|
|
1921
|
-
if (this._compressionLevel != null && this._compressFn) {
|
|
1922
|
-
return await this._compressFn(content, this._compressionLevel);
|
|
1739
|
+
async _readOutputBatch() {
|
|
1740
|
+
while (true) {
|
|
1741
|
+
const batch = await this._reader.readNextBatch();
|
|
1742
|
+
if (batch === null)
|
|
1743
|
+
return null;
|
|
1744
|
+
if (batch.numRows === 0) {
|
|
1745
|
+
if (isExternalLocationBatch(batch)) {
|
|
1746
|
+
return await resolveExternalLocation(batch, this._externalConfig);
|
|
1747
|
+
}
|
|
1748
|
+
if (dispatchLogOrError(batch, this._onLog)) {
|
|
1749
|
+
continue;
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
return batch;
|
|
1923
1753
|
}
|
|
1924
|
-
return content;
|
|
1925
1754
|
}
|
|
1926
|
-
async
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1755
|
+
async _ensureOutputStream() {
|
|
1756
|
+
if (this._outputStreamOpened)
|
|
1757
|
+
return;
|
|
1758
|
+
this._outputStreamOpened = true;
|
|
1759
|
+
const schema2 = await this._reader.openNextStream();
|
|
1760
|
+
if (!schema2) {
|
|
1761
|
+
throw new RpcError("ProtocolError", "Expected output stream but got EOF", "");
|
|
1930
1762
|
}
|
|
1931
|
-
return body;
|
|
1932
1763
|
}
|
|
1933
1764
|
async exchange(input) {
|
|
1934
|
-
if (this.
|
|
1935
|
-
throw new RpcError("ProtocolError", "Stream
|
|
1765
|
+
if (this._closed) {
|
|
1766
|
+
throw new RpcError("ProtocolError", "Stream session is closed", "");
|
|
1936
1767
|
}
|
|
1768
|
+
let inputSchema;
|
|
1769
|
+
let batch;
|
|
1937
1770
|
if (input.length === 0) {
|
|
1938
|
-
|
|
1939
|
-
const
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
const
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1771
|
+
inputSchema = this._inputSchema ?? this._outputSchema;
|
|
1772
|
+
const children = inputSchema.fields.map((f) => {
|
|
1773
|
+
return makeData({ type: f.type, length: 0, nullCount: 0 });
|
|
1774
|
+
});
|
|
1775
|
+
const structType = new Struct(inputSchema.fields);
|
|
1776
|
+
const data = makeData({
|
|
1777
|
+
type: structType,
|
|
1778
|
+
length: 0,
|
|
1779
|
+
children,
|
|
1780
|
+
nullCount: 0
|
|
1781
|
+
});
|
|
1782
|
+
batch = new RecordBatch(inputSchema, data);
|
|
1783
|
+
} else {
|
|
1784
|
+
const keys = Object.keys(input[0]);
|
|
1785
|
+
const fields = keys.map((key) => {
|
|
1786
|
+
let sample;
|
|
1787
|
+
for (const row of input) {
|
|
1788
|
+
if (row[key] != null) {
|
|
1789
|
+
sample = row[key];
|
|
1790
|
+
break;
|
|
1791
|
+
}
|
|
1952
1792
|
}
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
return vectorFromArray(values, f.type).data[0];
|
|
1962
|
-
});
|
|
1963
|
-
const structType = new Struct(inputSchema.fields);
|
|
1964
|
-
const data = makeData({
|
|
1965
|
-
type: structType,
|
|
1966
|
-
length: input.length,
|
|
1967
|
-
children,
|
|
1968
|
-
nullCount: 0
|
|
1969
|
-
});
|
|
1970
|
-
const metadata = new Map;
|
|
1971
|
-
metadata.set(STATE_KEY, this._stateToken);
|
|
1972
|
-
const batch = new RecordBatch(inputSchema, data, metadata);
|
|
1973
|
-
return this._doExchange(inputSchema, [batch]);
|
|
1974
|
-
}
|
|
1975
|
-
async _doExchange(schema2, batches) {
|
|
1976
|
-
const body = serializeIpcStream(schema2, batches);
|
|
1977
|
-
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
1978
|
-
if (resp.status === 401) {
|
|
1979
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
1980
|
-
}
|
|
1981
|
-
const responseBody = await this._readResponse(resp);
|
|
1982
|
-
const { batches: responseBatches } = await readResponseBatches(responseBody);
|
|
1983
|
-
let resultRows = [];
|
|
1984
|
-
for (const batch of responseBatches) {
|
|
1985
|
-
if (batch.numRows === 0) {
|
|
1986
|
-
dispatchLogOrError(batch, this._onLog);
|
|
1987
|
-
const token2 = batch.metadata?.get(STATE_KEY);
|
|
1988
|
-
if (token2) {
|
|
1989
|
-
this._stateToken = token2;
|
|
1793
|
+
const arrowType = inferArrowType(sample);
|
|
1794
|
+
return new Field(key, arrowType, true);
|
|
1795
|
+
});
|
|
1796
|
+
inputSchema = new Schema(fields);
|
|
1797
|
+
if (this._inputSchema) {
|
|
1798
|
+
const cached = this._inputSchema;
|
|
1799
|
+
if (cached.fields.length !== inputSchema.fields.length || cached.fields.some((f, i) => f.name !== inputSchema.fields[i].name)) {
|
|
1800
|
+
throw new RpcError("ProtocolError", `Exchange input schema changed: expected [${cached.fields.map((f) => f.name).join(", ")}] ` + `but got [${inputSchema.fields.map((f) => f.name).join(", ")}]`, "");
|
|
1990
1801
|
}
|
|
1991
|
-
|
|
1802
|
+
} else {
|
|
1803
|
+
this._inputSchema = inputSchema;
|
|
1992
1804
|
}
|
|
1993
|
-
const
|
|
1994
|
-
|
|
1995
|
-
|
|
1805
|
+
const children = inputSchema.fields.map((f) => {
|
|
1806
|
+
const values = input.map((row) => row[f.name]);
|
|
1807
|
+
return vectorFromArray(values, f.type).data[0];
|
|
1808
|
+
});
|
|
1809
|
+
const structType = new Struct(inputSchema.fields);
|
|
1810
|
+
const data = makeData({
|
|
1811
|
+
type: structType,
|
|
1812
|
+
length: input.length,
|
|
1813
|
+
children,
|
|
1814
|
+
nullCount: 0
|
|
1815
|
+
});
|
|
1816
|
+
batch = new RecordBatch(inputSchema, data);
|
|
1817
|
+
}
|
|
1818
|
+
if (!this._inputWriter) {
|
|
1819
|
+
this._inputWriter = new PipeIncrementalWriter(this._writeFn, inputSchema);
|
|
1820
|
+
}
|
|
1821
|
+
this._inputWriter.write(batch);
|
|
1822
|
+
await this._ensureOutputStream();
|
|
1823
|
+
try {
|
|
1824
|
+
const outputBatch = await this._readOutputBatch();
|
|
1825
|
+
if (outputBatch === null) {
|
|
1826
|
+
return [];
|
|
1996
1827
|
}
|
|
1997
|
-
|
|
1828
|
+
return extractBatchRows(outputBatch);
|
|
1829
|
+
} catch (e) {
|
|
1830
|
+
await this._cleanup();
|
|
1831
|
+
throw e;
|
|
1998
1832
|
}
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
1833
|
+
}
|
|
1834
|
+
async _cleanup() {
|
|
1835
|
+
if (this._closed)
|
|
1836
|
+
return;
|
|
1837
|
+
this._closed = true;
|
|
1838
|
+
if (this._inputWriter) {
|
|
1839
|
+
this._inputWriter.close();
|
|
1840
|
+
this._inputWriter = null;
|
|
1841
|
+
}
|
|
1842
|
+
try {
|
|
1843
|
+
if (this._outputStreamOpened) {
|
|
1844
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
1845
|
+
}
|
|
1846
|
+
} catch {}
|
|
1847
|
+
this._releaseBusy();
|
|
2013
1848
|
}
|
|
2014
1849
|
async* [Symbol.asyncIterator]() {
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
1850
|
+
if (this._closed)
|
|
1851
|
+
return;
|
|
1852
|
+
try {
|
|
1853
|
+
const tickSchema = new Schema([]);
|
|
1854
|
+
this._inputWriter = new PipeIncrementalWriter(this._writeFn, tickSchema);
|
|
1855
|
+
const structType = new Struct(tickSchema.fields);
|
|
1856
|
+
const tickData = makeData({
|
|
1857
|
+
type: structType,
|
|
1858
|
+
length: 0,
|
|
1859
|
+
children: [],
|
|
1860
|
+
nullCount: 0
|
|
1861
|
+
});
|
|
1862
|
+
const tickBatch = new RecordBatch(tickSchema, tickData);
|
|
1863
|
+
while (true) {
|
|
1864
|
+
this._inputWriter.write(tickBatch);
|
|
1865
|
+
await this._ensureOutputStream();
|
|
1866
|
+
const outputBatch = await this._readOutputBatch();
|
|
1867
|
+
if (outputBatch === null) {
|
|
1868
|
+
break;
|
|
2022
1869
|
}
|
|
1870
|
+
yield extractBatchRows(outputBatch);
|
|
2023
1871
|
}
|
|
2024
|
-
|
|
1872
|
+
} finally {
|
|
1873
|
+
if (this._inputWriter) {
|
|
1874
|
+
this._inputWriter.close();
|
|
1875
|
+
this._inputWriter = null;
|
|
1876
|
+
}
|
|
1877
|
+
try {
|
|
1878
|
+
if (this._outputStreamOpened) {
|
|
1879
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
1880
|
+
}
|
|
1881
|
+
} catch {}
|
|
1882
|
+
this._closed = true;
|
|
1883
|
+
this._releaseBusy();
|
|
2025
1884
|
}
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
if (this._stateToken === null)
|
|
1885
|
+
}
|
|
1886
|
+
close() {
|
|
1887
|
+
if (this._closed)
|
|
2030
1888
|
return;
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
const
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
if (isExternalLocationBatch(batch)) {
|
|
2047
|
-
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2048
|
-
} else {
|
|
2049
|
-
dispatchLogOrError(batch, this._onLog);
|
|
2050
|
-
continue;
|
|
1889
|
+
this._closed = true;
|
|
1890
|
+
if (this._inputWriter) {
|
|
1891
|
+
this._inputWriter.close();
|
|
1892
|
+
this._inputWriter = null;
|
|
1893
|
+
} else {
|
|
1894
|
+
const emptySchema = new Schema([]);
|
|
1895
|
+
const ipc = serializeIpcStream(emptySchema, []);
|
|
1896
|
+
this._writeFn(ipc);
|
|
1897
|
+
}
|
|
1898
|
+
const drainPromise = (async () => {
|
|
1899
|
+
try {
|
|
1900
|
+
if (!this._outputStreamOpened) {
|
|
1901
|
+
const schema2 = await this._reader.openNextStream();
|
|
1902
|
+
if (schema2) {
|
|
1903
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
2051
1904
|
}
|
|
1905
|
+
} else {
|
|
1906
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
2052
1907
|
}
|
|
2053
|
-
|
|
1908
|
+
} catch {} finally {
|
|
1909
|
+
this._releaseBusy();
|
|
2054
1910
|
}
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
}
|
|
1911
|
+
})();
|
|
1912
|
+
this._setDrainPromise(drainPromise);
|
|
2058
1913
|
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
1914
|
+
}
|
|
1915
|
+
function pipeConnect(readable, writable, options) {
|
|
1916
|
+
const onLog = options?.onLog;
|
|
1917
|
+
const externalConfig = options?.externalLocation;
|
|
1918
|
+
let reader = null;
|
|
1919
|
+
let readerPromise = null;
|
|
1920
|
+
let methodCache = null;
|
|
1921
|
+
let protocolName = "";
|
|
1922
|
+
let serverProtocolVersion = "";
|
|
1923
|
+
let _busy = false;
|
|
1924
|
+
let _drainPromise = null;
|
|
1925
|
+
let closed = false;
|
|
1926
|
+
const writeFn = (bytes) => {
|
|
1927
|
+
writable.write(bytes);
|
|
1928
|
+
writable.flush?.();
|
|
1929
|
+
};
|
|
1930
|
+
async function ensureReader() {
|
|
1931
|
+
if (reader)
|
|
1932
|
+
return reader;
|
|
1933
|
+
if (!readerPromise) {
|
|
1934
|
+
readerPromise = IpcStreamReader.create(readable);
|
|
2075
1935
|
}
|
|
2076
|
-
|
|
1936
|
+
reader = await readerPromise;
|
|
1937
|
+
return reader;
|
|
2077
1938
|
}
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
const headers = { "Content-Type": ARROW_CONTENT_TYPE };
|
|
2088
|
-
if (authorization)
|
|
2089
|
-
headers.Authorization = authorization;
|
|
2090
|
-
const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
|
|
2091
|
-
method: "POST",
|
|
2092
|
-
headers,
|
|
2093
|
-
body
|
|
2094
|
-
});
|
|
2095
|
-
if (resp.status === 404) {
|
|
2096
|
-
throw new RpcError("NotSupported", "Server does not support upload URLs", "");
|
|
1939
|
+
async function acquireBusy() {
|
|
1940
|
+
if (_drainPromise) {
|
|
1941
|
+
await _drainPromise;
|
|
1942
|
+
_drainPromise = null;
|
|
1943
|
+
}
|
|
1944
|
+
if (_busy) {
|
|
1945
|
+
throw new Error("Pipe transport is busy — another call or stream is in progress. " + "Pipe connections are single-threaded; wait for the current operation to complete.");
|
|
1946
|
+
}
|
|
1947
|
+
_busy = true;
|
|
2097
1948
|
}
|
|
2098
|
-
|
|
2099
|
-
|
|
1949
|
+
function releaseBusy() {
|
|
1950
|
+
_busy = false;
|
|
2100
1951
|
}
|
|
2101
|
-
|
|
2102
|
-
|
|
1952
|
+
function setDrainPromise(p) {
|
|
1953
|
+
_drainPromise = p;
|
|
2103
1954
|
}
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
const
|
|
2113
|
-
const
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
if (expiresRaw instanceof Date) {
|
|
2117
|
-
expiresAt = expiresRaw;
|
|
2118
|
-
} else if (typeof expiresRaw === "bigint") {
|
|
2119
|
-
expiresAt = new Date(Number(expiresRaw / 1000n));
|
|
2120
|
-
} else if (typeof expiresRaw === "number") {
|
|
2121
|
-
expiresAt = new Date(expiresRaw);
|
|
2122
|
-
} else {
|
|
2123
|
-
expiresAt = new Date;
|
|
1955
|
+
async function ensureMethodCache() {
|
|
1956
|
+
if (methodCache)
|
|
1957
|
+
return methodCache;
|
|
1958
|
+
await acquireBusy();
|
|
1959
|
+
try {
|
|
1960
|
+
const emptySchema = new Schema([]);
|
|
1961
|
+
const body = buildRequestIpc(emptySchema, {}, DESCRIBE_METHOD_NAME);
|
|
1962
|
+
writeFn(body);
|
|
1963
|
+
const r = await ensureReader();
|
|
1964
|
+
const response = await r.readStream();
|
|
1965
|
+
if (!response) {
|
|
1966
|
+
throw new Error("EOF reading __describe__ response");
|
|
2124
1967
|
}
|
|
2125
|
-
|
|
1968
|
+
const desc = await parseDescribeResponse(response.batches, onLog);
|
|
1969
|
+
protocolName = desc.protocolName;
|
|
1970
|
+
serverProtocolVersion = desc.protocolVersion;
|
|
1971
|
+
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
1972
|
+
return methodCache;
|
|
1973
|
+
} finally {
|
|
1974
|
+
releaseBusy();
|
|
2126
1975
|
}
|
|
2127
1976
|
}
|
|
2128
|
-
if (pairs.length === 0) {
|
|
2129
|
-
throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
|
|
2130
|
-
}
|
|
2131
|
-
return pairs;
|
|
2132
|
-
}
|
|
2133
|
-
async function buildPointerRequestBody(originalBody, downloadUrl) {
|
|
2134
|
-
const reader = await RecordBatchReader4.from(originalBody);
|
|
2135
|
-
await reader.open();
|
|
2136
|
-
const schema2 = reader.schema;
|
|
2137
|
-
if (!schema2) {
|
|
2138
|
-
throw new RpcError("ProtocolError", "Original request body has no schema", "");
|
|
2139
|
-
}
|
|
2140
|
-
const batches = reader.readAll();
|
|
2141
|
-
if (batches.length === 0) {
|
|
2142
|
-
throw new RpcError("ProtocolError", "Original request body has no batches", "");
|
|
2143
|
-
}
|
|
2144
|
-
const original = batches[0];
|
|
2145
|
-
const originalMeta = original.metadata ?? new Map;
|
|
2146
|
-
const pointer = makeExternalLocationBatch(schema2, downloadUrl);
|
|
2147
|
-
const merged = new Map(pointer.metadata ?? new Map);
|
|
2148
|
-
const method = originalMeta.get(RPC_METHOD_KEY);
|
|
2149
|
-
const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
|
|
2150
|
-
if (method)
|
|
2151
|
-
merged.set(RPC_METHOD_KEY, method);
|
|
2152
|
-
merged.set(REQUEST_VERSION_KEY, version);
|
|
2153
|
-
for (const [k, v] of originalMeta) {
|
|
2154
|
-
if (!merged.has(k))
|
|
2155
|
-
merged.set(k, v);
|
|
2156
|
-
}
|
|
2157
|
-
const { RecordBatch: RecordBatch2 } = await import("@query-farm/apache-arrow");
|
|
2158
|
-
const pointerWithMeta = new RecordBatch2(schema2, pointer.data, merged);
|
|
2159
|
-
return serializeIpcStream(schema2, [pointerWithMeta]);
|
|
2160
|
-
}
|
|
2161
|
-
async function externalizeRequestBody(body, opts) {
|
|
2162
|
-
const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
|
|
2163
|
-
const pair = pairs[0];
|
|
2164
|
-
if (opts.urlValidator) {
|
|
2165
|
-
opts.urlValidator(pair.uploadUrl);
|
|
2166
|
-
opts.urlValidator(pair.downloadUrl);
|
|
2167
|
-
}
|
|
2168
|
-
const putResp = await fetch(pair.uploadUrl, {
|
|
2169
|
-
method: "PUT",
|
|
2170
|
-
headers: { "Content-Type": ARROW_CONTENT_TYPE },
|
|
2171
|
-
body
|
|
2172
|
-
});
|
|
2173
|
-
if (!putResp.ok) {
|
|
2174
|
-
throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
|
|
2175
|
-
}
|
|
2176
|
-
return buildPointerRequestBody(body, pair.downloadUrl);
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
// src/client/connect.ts
|
|
2180
|
-
function httpConnect(baseUrl, options) {
|
|
2181
|
-
const prefix = (options?.prefix ?? "").replace(/\/+$/, "");
|
|
2182
|
-
const onLog = options?.onLog;
|
|
2183
|
-
const compressionLevel = options?.compressionLevel;
|
|
2184
|
-
const authorization = options?.authorization;
|
|
2185
|
-
const externalConfig = options?.externalLocation;
|
|
2186
|
-
let methodCache = null;
|
|
2187
|
-
let serverProtocolVersion = "";
|
|
2188
|
-
let compressFn;
|
|
2189
|
-
let decompressFn;
|
|
2190
|
-
let compressionLoaded = false;
|
|
2191
|
-
let capabilities = null;
|
|
2192
|
-
function updateCapabilitiesFromResponse(resp) {
|
|
2193
|
-
const next = parseCapabilitiesFromHeaders(resp.headers);
|
|
2194
|
-
if (next.maxRequestBytes != null || next.uploadUrlSupport) {
|
|
2195
|
-
capabilities = next;
|
|
2196
|
-
}
|
|
2197
|
-
}
|
|
2198
|
-
async function maybeExternalize(body) {
|
|
2199
|
-
const caps = isCapabilitySnapshotFresh(capabilities) ? capabilities : null;
|
|
2200
|
-
if (!caps)
|
|
2201
|
-
return body;
|
|
2202
|
-
if (!caps.uploadUrlSupport)
|
|
2203
|
-
return body;
|
|
2204
|
-
if (caps.maxRequestBytes == null || body.byteLength <= caps.maxRequestBytes)
|
|
2205
|
-
return body;
|
|
2206
|
-
return externalizeRequestBody(body, {
|
|
2207
|
-
baseUrl,
|
|
2208
|
-
prefix,
|
|
2209
|
-
authorization,
|
|
2210
|
-
urlValidator: externalConfig?.urlValidator ?? null
|
|
2211
|
-
});
|
|
2212
|
-
}
|
|
2213
|
-
async function postWithExternalization(url, body) {
|
|
2214
|
-
const sendBody = await maybeExternalize(body);
|
|
2215
|
-
let resp = await fetch(url, {
|
|
2216
|
-
method: "POST",
|
|
2217
|
-
headers: buildHeaders(),
|
|
2218
|
-
body: await prepareBody(sendBody)
|
|
2219
|
-
});
|
|
2220
|
-
updateCapabilitiesFromResponse(resp);
|
|
2221
|
-
if (resp.status === 413 && capabilities?.uploadUrlSupport && body.byteLength > 0) {
|
|
2222
|
-
const externalized = await externalizeRequestBody(body, {
|
|
2223
|
-
baseUrl,
|
|
2224
|
-
prefix,
|
|
2225
|
-
authorization,
|
|
2226
|
-
urlValidator: externalConfig?.urlValidator ?? null
|
|
2227
|
-
});
|
|
2228
|
-
resp = await fetch(url, {
|
|
2229
|
-
method: "POST",
|
|
2230
|
-
headers: buildHeaders(),
|
|
2231
|
-
body: await prepareBody(externalized)
|
|
2232
|
-
});
|
|
2233
|
-
updateCapabilitiesFromResponse(resp);
|
|
2234
|
-
}
|
|
2235
|
-
return resp;
|
|
2236
|
-
}
|
|
2237
|
-
async function ensureCompression() {
|
|
2238
|
-
if (compressionLoaded || compressionLevel == null)
|
|
2239
|
-
return;
|
|
2240
|
-
try {
|
|
2241
|
-
const mod = await Promise.resolve().then(() => (init_zstd(), exports_zstd));
|
|
2242
|
-
compressFn = mod.zstdCompress;
|
|
2243
|
-
decompressFn = mod.zstdDecompress;
|
|
2244
|
-
} catch {}
|
|
2245
|
-
compressionLoaded = true;
|
|
2246
|
-
}
|
|
2247
|
-
function buildHeaders() {
|
|
2248
|
-
const headers = {
|
|
2249
|
-
"Content-Type": ARROW_CONTENT_TYPE
|
|
2250
|
-
};
|
|
2251
|
-
if (compressionLevel != null && compressFn) {
|
|
2252
|
-
headers["Content-Encoding"] = "zstd";
|
|
2253
|
-
}
|
|
2254
|
-
if (compressionLevel != null && decompressFn) {
|
|
2255
|
-
headers["Accept-Encoding"] = "zstd";
|
|
2256
|
-
}
|
|
2257
|
-
if (authorization) {
|
|
2258
|
-
headers.Authorization = authorization;
|
|
2259
|
-
}
|
|
2260
|
-
return headers;
|
|
2261
|
-
}
|
|
2262
|
-
async function prepareBody(content) {
|
|
2263
|
-
if (compressionLevel != null && compressFn) {
|
|
2264
|
-
return await compressFn(content, compressionLevel);
|
|
2265
|
-
}
|
|
2266
|
-
return content;
|
|
2267
|
-
}
|
|
2268
|
-
function checkAuth(resp) {
|
|
2269
|
-
if (resp.status === 401) {
|
|
2270
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2271
|
-
}
|
|
2272
|
-
}
|
|
2273
|
-
async function readResponse(resp) {
|
|
2274
|
-
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2275
|
-
if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
|
|
2276
|
-
body = new Uint8Array(await decompressFn(body));
|
|
2277
|
-
}
|
|
2278
|
-
return body;
|
|
2279
|
-
}
|
|
2280
|
-
async function ensureMethodCache() {
|
|
2281
|
-
if (methodCache)
|
|
2282
|
-
return methodCache;
|
|
2283
|
-
await ensureCompression();
|
|
2284
|
-
const desc = await httpIntrospect(baseUrl, {
|
|
2285
|
-
prefix,
|
|
2286
|
-
authorization,
|
|
2287
|
-
compressionLevel,
|
|
2288
|
-
compressFn,
|
|
2289
|
-
decompressFn
|
|
2290
|
-
});
|
|
2291
|
-
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2292
|
-
serverProtocolVersion = desc.protocolVersion;
|
|
2293
|
-
return methodCache;
|
|
2294
|
-
}
|
|
2295
1977
|
return {
|
|
2296
1978
|
async call(method, params) {
|
|
2297
|
-
await ensureCompression();
|
|
2298
1979
|
const methods = await ensureMethodCache();
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2305
|
-
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}`, body);
|
|
2306
|
-
checkAuth(resp);
|
|
2307
|
-
const responseBody = await readResponse(resp);
|
|
2308
|
-
const { batches } = await readResponseBatches(responseBody);
|
|
2309
|
-
let resultBatch = null;
|
|
2310
|
-
for (let batch of batches) {
|
|
2311
|
-
if (batch.numRows === 0) {
|
|
2312
|
-
if (isExternalLocationBatch(batch)) {
|
|
2313
|
-
batch = await resolveExternalLocation(batch, externalConfig);
|
|
2314
|
-
} else {
|
|
2315
|
-
dispatchLogOrError(batch, onLog);
|
|
2316
|
-
continue;
|
|
2317
|
-
}
|
|
1980
|
+
await acquireBusy();
|
|
1981
|
+
try {
|
|
1982
|
+
const info = methods.get(method);
|
|
1983
|
+
if (!info) {
|
|
1984
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2318
1985
|
}
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
await ensureCompression();
|
|
2334
|
-
const methods = await ensureMethodCache();
|
|
2335
|
-
const info = methods.get(method);
|
|
2336
|
-
if (!info) {
|
|
2337
|
-
throw new Error(`Unknown method: '${method}'`);
|
|
2338
|
-
}
|
|
2339
|
-
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2340
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2341
|
-
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}/init`, body);
|
|
2342
|
-
checkAuth(resp);
|
|
2343
|
-
const responseBody = await readResponse(resp);
|
|
2344
|
-
let header = null;
|
|
2345
|
-
let stateToken = null;
|
|
2346
|
-
const pendingBatches = [];
|
|
2347
|
-
let finished = false;
|
|
2348
|
-
let streamSchema = null;
|
|
2349
|
-
if (info.headerSchema) {
|
|
2350
|
-
const reader = await readSequentialStreams(responseBody);
|
|
2351
|
-
const headerStream = await reader.readStream();
|
|
2352
|
-
if (headerStream) {
|
|
2353
|
-
for (const batch of headerStream.batches) {
|
|
2354
|
-
if (batch.numRows === 0) {
|
|
1986
|
+
const r = await ensureReader();
|
|
1987
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
1988
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
1989
|
+
writeFn(body);
|
|
1990
|
+
const response = await r.readStream();
|
|
1991
|
+
if (!response) {
|
|
1992
|
+
throw new Error("EOF reading response");
|
|
1993
|
+
}
|
|
1994
|
+
let resultBatch = null;
|
|
1995
|
+
for (let batch of response.batches) {
|
|
1996
|
+
if (batch.numRows === 0) {
|
|
1997
|
+
if (isExternalLocationBatch(batch)) {
|
|
1998
|
+
batch = await resolveExternalLocation(batch, externalConfig);
|
|
1999
|
+
} else {
|
|
2355
2000
|
dispatchLogOrError(batch, onLog);
|
|
2356
2001
|
continue;
|
|
2357
2002
|
}
|
|
2358
|
-
const rows = extractBatchRows(batch);
|
|
2359
|
-
if (rows.length > 0) {
|
|
2360
|
-
header = rows[0];
|
|
2361
|
-
}
|
|
2362
2003
|
}
|
|
2004
|
+
resultBatch = batch;
|
|
2363
2005
|
}
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
streamSchema = dataStream.schema;
|
|
2006
|
+
if (!resultBatch) {
|
|
2007
|
+
return null;
|
|
2367
2008
|
}
|
|
2368
|
-
const
|
|
2369
|
-
if (
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2009
|
+
const rows = extractBatchRows(resultBatch);
|
|
2010
|
+
if (rows.length === 0)
|
|
2011
|
+
return null;
|
|
2012
|
+
if (info.resultSchema.fields.length === 0)
|
|
2013
|
+
return null;
|
|
2014
|
+
return rows[0];
|
|
2015
|
+
} finally {
|
|
2016
|
+
releaseBusy();
|
|
2017
|
+
}
|
|
2018
|
+
},
|
|
2019
|
+
async stream(method, params) {
|
|
2020
|
+
const methods = await ensureMethodCache();
|
|
2021
|
+
await acquireBusy();
|
|
2022
|
+
try {
|
|
2023
|
+
const info = methods.get(method);
|
|
2024
|
+
if (!info) {
|
|
2025
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2026
|
+
}
|
|
2027
|
+
const r = await ensureReader();
|
|
2028
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2029
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2030
|
+
writeFn(body);
|
|
2031
|
+
let header = null;
|
|
2032
|
+
if (info.headerSchema) {
|
|
2033
|
+
const headerStream = await r.readStream();
|
|
2034
|
+
if (headerStream) {
|
|
2035
|
+
for (const batch of headerStream.batches) {
|
|
2036
|
+
if (batch.numRows === 0) {
|
|
2037
|
+
dispatchLogOrError(batch, onLog);
|
|
2375
2038
|
continue;
|
|
2376
2039
|
}
|
|
2377
|
-
const
|
|
2378
|
-
if (
|
|
2379
|
-
|
|
2380
|
-
continue;
|
|
2040
|
+
const rows = extractBatchRows(batch);
|
|
2041
|
+
if (rows.length > 0) {
|
|
2042
|
+
header = rows[0];
|
|
2381
2043
|
}
|
|
2382
|
-
dispatchLogOrError(batch, onLog);
|
|
2383
|
-
continue;
|
|
2384
2044
|
}
|
|
2385
|
-
pendingBatches.push(batch);
|
|
2386
2045
|
}
|
|
2387
2046
|
}
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
continue;
|
|
2410
|
-
}
|
|
2411
|
-
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2412
|
-
if (level === "EXCEPTION") {
|
|
2413
|
-
errorBatches.push(batch);
|
|
2414
|
-
continue;
|
|
2415
|
-
}
|
|
2416
|
-
dispatchLogOrError(batch, onLog);
|
|
2417
|
-
continue;
|
|
2418
|
-
}
|
|
2419
|
-
pendingBatches.push(batch);
|
|
2420
|
-
}
|
|
2421
|
-
if (errorBatches.length > 0) {
|
|
2422
|
-
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2423
|
-
pendingBatches.push(...errorBatches);
|
|
2424
|
-
} else {
|
|
2425
|
-
for (const batch of errorBatches) {
|
|
2426
|
-
dispatchLogOrError(batch, onLog);
|
|
2427
|
-
}
|
|
2428
|
-
}
|
|
2429
|
-
}
|
|
2430
|
-
}
|
|
2431
|
-
if (pendingBatches.length === 0 && stateToken === null) {
|
|
2432
|
-
finished = true;
|
|
2047
|
+
const outputSchema = info.outputSchema ?? info.resultSchema;
|
|
2048
|
+
return new PipeStreamSession({
|
|
2049
|
+
reader: r,
|
|
2050
|
+
writeFn,
|
|
2051
|
+
onLog,
|
|
2052
|
+
header,
|
|
2053
|
+
outputSchema,
|
|
2054
|
+
releaseBusy,
|
|
2055
|
+
setDrainPromise,
|
|
2056
|
+
externalConfig
|
|
2057
|
+
});
|
|
2058
|
+
} catch (e) {
|
|
2059
|
+
try {
|
|
2060
|
+
const r = await ensureReader();
|
|
2061
|
+
const emptySchema = new Schema([]);
|
|
2062
|
+
const ipc = serializeIpcStream(emptySchema, []);
|
|
2063
|
+
writeFn(ipc);
|
|
2064
|
+
const outStream = await r.readStream();
|
|
2065
|
+
} catch {}
|
|
2066
|
+
releaseBusy();
|
|
2067
|
+
throw e;
|
|
2433
2068
|
}
|
|
2434
|
-
const outputSchema = (streamSchema && streamSchema.fields.length > 0 ? streamSchema : null) ?? (pendingBatches.length > 0 ? pendingBatches[0].schema : null) ?? info.outputSchema ?? info.resultSchema;
|
|
2435
|
-
return new HttpStreamSession({
|
|
2436
|
-
baseUrl,
|
|
2437
|
-
prefix,
|
|
2438
|
-
method,
|
|
2439
|
-
stateToken,
|
|
2440
|
-
outputSchema,
|
|
2441
|
-
inputSchema: info.inputSchema,
|
|
2442
|
-
onLog,
|
|
2443
|
-
pendingBatches,
|
|
2444
|
-
finished,
|
|
2445
|
-
header,
|
|
2446
|
-
compressionLevel,
|
|
2447
|
-
compressFn,
|
|
2448
|
-
decompressFn,
|
|
2449
|
-
authorization,
|
|
2450
|
-
externalConfig,
|
|
2451
|
-
postFn: postWithExternalization
|
|
2452
|
-
});
|
|
2453
2069
|
},
|
|
2454
2070
|
async describe() {
|
|
2455
|
-
await
|
|
2456
|
-
return
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
decompressFn
|
|
2462
|
-
});
|
|
2071
|
+
const methods = await ensureMethodCache();
|
|
2072
|
+
return {
|
|
2073
|
+
protocolName,
|
|
2074
|
+
protocolVersion: serverProtocolVersion,
|
|
2075
|
+
methods: [...methods.values()]
|
|
2076
|
+
};
|
|
2463
2077
|
},
|
|
2464
|
-
close() {
|
|
2078
|
+
close() {
|
|
2079
|
+
if (closed)
|
|
2080
|
+
return;
|
|
2081
|
+
closed = true;
|
|
2082
|
+
writable.end();
|
|
2083
|
+
}
|
|
2465
2084
|
};
|
|
2466
2085
|
}
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2086
|
+
function subprocessConnect(cmd, options) {
|
|
2087
|
+
const proc = Bun.spawn(cmd, {
|
|
2088
|
+
stdin: "pipe",
|
|
2089
|
+
stdout: "pipe",
|
|
2090
|
+
stderr: options?.stderr ?? "ignore",
|
|
2091
|
+
cwd: options?.cwd,
|
|
2092
|
+
env: options?.env ? { ...process.env, ...options.env } : undefined
|
|
2093
|
+
});
|
|
2094
|
+
const stdout = proc.stdout;
|
|
2095
|
+
const writable = {
|
|
2096
|
+
write(data) {
|
|
2097
|
+
proc.stdin.write(data);
|
|
2098
|
+
},
|
|
2099
|
+
flush() {
|
|
2100
|
+
proc.stdin.flush();
|
|
2101
|
+
},
|
|
2102
|
+
end() {
|
|
2103
|
+
proc.stdin.end();
|
|
2104
|
+
}
|
|
2472
2105
|
};
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
if (json.resource_tos_uri)
|
|
2486
|
-
result.resourceTosUri = json.resource_tos_uri;
|
|
2487
|
-
if (json.client_id)
|
|
2488
|
-
result.clientId = json.client_id;
|
|
2489
|
-
if (json.client_secret)
|
|
2490
|
-
result.clientSecret = json.client_secret;
|
|
2491
|
-
if (json.use_id_token_as_bearer)
|
|
2492
|
-
result.useIdTokenAsBearer = json.use_id_token_as_bearer;
|
|
2493
|
-
if (json.device_code_client_id)
|
|
2494
|
-
result.deviceCodeClientId = json.device_code_client_id;
|
|
2495
|
-
if (json.device_code_client_secret)
|
|
2496
|
-
result.deviceCodeClientSecret = json.device_code_client_secret;
|
|
2497
|
-
return result;
|
|
2106
|
+
const client = pipeConnect(stdout, writable, {
|
|
2107
|
+
onLog: options?.onLog,
|
|
2108
|
+
externalLocation: options?.externalLocation
|
|
2109
|
+
});
|
|
2110
|
+
const originalClose = client.close;
|
|
2111
|
+
client.close = () => {
|
|
2112
|
+
originalClose.call(client);
|
|
2113
|
+
try {
|
|
2114
|
+
proc.kill();
|
|
2115
|
+
} catch {}
|
|
2116
|
+
};
|
|
2117
|
+
return client;
|
|
2498
2118
|
}
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2119
|
+
|
|
2120
|
+
// src/client/tcp.ts
|
|
2121
|
+
function tcpConnect(host, port, options) {
|
|
2122
|
+
const socket = connect({ host, port });
|
|
2123
|
+
socket.setNoDelay(true);
|
|
2124
|
+
socket.on("error", () => {});
|
|
2125
|
+
const readable = socket;
|
|
2126
|
+
const writable = {
|
|
2127
|
+
write(data) {
|
|
2128
|
+
socket.write(data);
|
|
2129
|
+
},
|
|
2130
|
+
end() {
|
|
2131
|
+
socket.end();
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
const client = pipeConnect(readable, writable, {
|
|
2135
|
+
onLog: options?.onLog,
|
|
2136
|
+
externalLocation: options?.externalLocation
|
|
2137
|
+
});
|
|
2138
|
+
const originalClose = client.close;
|
|
2139
|
+
client.close = () => {
|
|
2140
|
+
originalClose.call(client);
|
|
2141
|
+
try {
|
|
2142
|
+
socket.destroy();
|
|
2143
|
+
} catch {}
|
|
2144
|
+
};
|
|
2145
|
+
return client;
|
|
2507
2146
|
}
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2147
|
+
// src/access-log.ts
|
|
2148
|
+
var _NODE_FS_MOD = "node:fs";
|
|
2149
|
+
function _loadWriteSync() {
|
|
2150
|
+
const req = import.meta.require ?? globalThis.require ?? null;
|
|
2151
|
+
if (!req) {
|
|
2152
|
+
throw new Error("FdSink requires Node.js or Bun (node:fs.writeSync). For other runtimes, " + "supply a custom AccessLogSink that wraps console.log or your logger.");
|
|
2512
2153
|
}
|
|
2513
|
-
|
|
2514
|
-
return parseMetadataJson(json);
|
|
2515
|
-
}
|
|
2516
|
-
function parseResourceMetadataUrl(wwwAuthenticate) {
|
|
2517
|
-
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
2518
|
-
if (!bearerMatch)
|
|
2519
|
-
return null;
|
|
2520
|
-
const params = bearerMatch[1];
|
|
2521
|
-
const metadataMatch = params.match(/resource_metadata="([^"]+)"/);
|
|
2522
|
-
if (!metadataMatch)
|
|
2523
|
-
return null;
|
|
2524
|
-
return metadataMatch[1];
|
|
2154
|
+
return req(_NODE_FS_MOD).writeSync;
|
|
2525
2155
|
}
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2156
|
+
|
|
2157
|
+
class FdSink {
|
|
2158
|
+
fd;
|
|
2159
|
+
_writeSync = _loadWriteSync();
|
|
2160
|
+
constructor(fd) {
|
|
2161
|
+
this.fd = fd;
|
|
2162
|
+
}
|
|
2163
|
+
write(line) {
|
|
2164
|
+
const buf = new TextEncoder().encode(line);
|
|
2165
|
+
let offset = 0;
|
|
2166
|
+
while (offset < buf.length) {
|
|
2167
|
+
const n = this._writeSync(this.fd, buf, offset, buf.length - offset);
|
|
2168
|
+
if (n <= 0)
|
|
2169
|
+
throw new Error(`access-log writeSync returned ${n}`);
|
|
2170
|
+
offset += n;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2535
2173
|
}
|
|
2536
|
-
function
|
|
2537
|
-
const
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
const
|
|
2541
|
-
const
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2174
|
+
function rfc3339Utc() {
|
|
2175
|
+
const d = new Date;
|
|
2176
|
+
const yyyy = d.getUTCFullYear().toString().padStart(4, "0");
|
|
2177
|
+
const mm = (d.getUTCMonth() + 1).toString().padStart(2, "0");
|
|
2178
|
+
const dd = d.getUTCDate().toString().padStart(2, "0");
|
|
2179
|
+
const hh = d.getUTCHours().toString().padStart(2, "0");
|
|
2180
|
+
const mi = d.getUTCMinutes().toString().padStart(2, "0");
|
|
2181
|
+
const ss = d.getUTCSeconds().toString().padStart(2, "0");
|
|
2182
|
+
const ms = d.getUTCMilliseconds().toString().padStart(3, "0");
|
|
2183
|
+
return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}Z`;
|
|
2545
2184
|
}
|
|
2546
|
-
function
|
|
2547
|
-
|
|
2548
|
-
if (!bearerMatch)
|
|
2549
|
-
return false;
|
|
2550
|
-
const params = bearerMatch[1];
|
|
2551
|
-
const match = params.match(/use_id_token_as_bearer="([^"]+)"/);
|
|
2552
|
-
if (!match)
|
|
2553
|
-
return false;
|
|
2554
|
-
return match[1] === "true";
|
|
2185
|
+
function base64(bytes) {
|
|
2186
|
+
return Buffer.from(bytes).toString("base64");
|
|
2555
2187
|
}
|
|
2556
|
-
function
|
|
2557
|
-
|
|
2558
|
-
if (!bearerMatch)
|
|
2559
|
-
return null;
|
|
2560
|
-
const params = bearerMatch[1];
|
|
2561
|
-
const match = params.match(/device_code_client_id="([^"]+)"/);
|
|
2562
|
-
if (!match)
|
|
2563
|
-
return null;
|
|
2564
|
-
return match[1];
|
|
2188
|
+
function roundTo2(f) {
|
|
2189
|
+
return Math.round(f * 100) / 100;
|
|
2565
2190
|
}
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2191
|
+
|
|
2192
|
+
class AccessLogHook {
|
|
2193
|
+
sink;
|
|
2194
|
+
serverVersion;
|
|
2195
|
+
level;
|
|
2196
|
+
constructor(sink, options = {}) {
|
|
2197
|
+
this.sink = sink;
|
|
2198
|
+
if (typeof options === "string") {
|
|
2199
|
+
this.serverVersion = options;
|
|
2200
|
+
this.level = "INFO";
|
|
2201
|
+
} else {
|
|
2202
|
+
this.serverVersion = options.serverVersion ?? "";
|
|
2203
|
+
this.level = options.level ?? "INFO";
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
onDispatchStart(_info) {
|
|
2207
|
+
const token = { startNs: process.hrtime.bigint() };
|
|
2208
|
+
return token;
|
|
2209
|
+
}
|
|
2210
|
+
onDispatchEnd(token, info, stats, error) {
|
|
2211
|
+
const t = token;
|
|
2212
|
+
const durationMs = t ? roundTo2(Number(process.hrtime.bigint() - t.startNs) / 1e6) : 0;
|
|
2213
|
+
const status = error ? "error" : "ok";
|
|
2214
|
+
const errType = error ? error.type ?? error.constructor.name : "";
|
|
2215
|
+
const errMsg = error?.message ?? "";
|
|
2216
|
+
const protocol = info.protocol ?? "";
|
|
2217
|
+
const rec = {
|
|
2218
|
+
timestamp: rfc3339Utc(),
|
|
2219
|
+
level: "INFO",
|
|
2220
|
+
logger: "vgi_rpc.access",
|
|
2221
|
+
message: `${protocol}.${info.method} ${status}`,
|
|
2222
|
+
server_id: info.serverId,
|
|
2223
|
+
protocol,
|
|
2224
|
+
protocol_hash: info.protocolHash ?? "",
|
|
2225
|
+
method: info.method,
|
|
2226
|
+
method_type: info.methodType,
|
|
2227
|
+
principal: info.principal ?? "",
|
|
2228
|
+
auth_domain: info.authDomain ?? "",
|
|
2229
|
+
authenticated: info.authenticated ?? false,
|
|
2230
|
+
remote_addr: info.remoteAddr ?? "",
|
|
2231
|
+
duration_ms: durationMs,
|
|
2232
|
+
status,
|
|
2233
|
+
error_type: errType
|
|
2234
|
+
};
|
|
2235
|
+
if (errMsg)
|
|
2236
|
+
rec.error_message = errMsg;
|
|
2237
|
+
if (this.serverVersion)
|
|
2238
|
+
rec.server_version = this.serverVersion;
|
|
2239
|
+
if (info.protocolVersion)
|
|
2240
|
+
rec.protocol_version = info.protocolVersion;
|
|
2241
|
+
if (info.requestId)
|
|
2242
|
+
rec.request_id = info.requestId;
|
|
2243
|
+
if (info.requestData && info.requestData.length > 0) {
|
|
2244
|
+
const encoded = base64(info.requestData);
|
|
2245
|
+
if (this.level === "DEBUG") {
|
|
2246
|
+
rec.request_data = encoded;
|
|
2247
|
+
} else {
|
|
2248
|
+
rec.original_request_bytes = encoded.length;
|
|
2249
|
+
rec.truncated = true;
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
if (info.methodType === "stream") {
|
|
2253
|
+
rec.stream_id = info.streamId ?? "00000000000000000000000000000000";
|
|
2254
|
+
}
|
|
2255
|
+
if (info.cancelled)
|
|
2256
|
+
rec.cancelled = true;
|
|
2257
|
+
if (stats.inputBatches + stats.outputBatches + stats.inputRows + stats.outputRows + stats.inputBytes + stats.outputBytes !== 0) {
|
|
2258
|
+
rec.input_batches = stats.inputBatches;
|
|
2259
|
+
rec.output_batches = stats.outputBatches;
|
|
2260
|
+
rec.input_rows = stats.inputRows;
|
|
2261
|
+
rec.output_rows = stats.outputRows;
|
|
2262
|
+
rec.input_bytes = stats.inputBytes;
|
|
2263
|
+
rec.output_bytes = stats.outputBytes;
|
|
2264
|
+
}
|
|
2265
|
+
try {
|
|
2266
|
+
this.sink.write(`${JSON.stringify(rec)}
|
|
2267
|
+
`);
|
|
2268
|
+
} catch {}
|
|
2269
|
+
}
|
|
2575
2270
|
}
|
|
2576
|
-
// src/
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
writer;
|
|
2588
|
-
writeFn;
|
|
2589
|
-
closed = false;
|
|
2590
|
-
constructor(writeFn, schema2) {
|
|
2591
|
-
this.writeFn = writeFn;
|
|
2592
|
-
this.writer = new RecordBatchStreamWriter2;
|
|
2593
|
-
this.writer.reset(undefined, schema2);
|
|
2594
|
-
this.drain();
|
|
2271
|
+
// src/auth.ts
|
|
2272
|
+
class AuthContext {
|
|
2273
|
+
domain;
|
|
2274
|
+
authenticated;
|
|
2275
|
+
principal;
|
|
2276
|
+
claims;
|
|
2277
|
+
constructor(domain, authenticated, principal, claims = {}) {
|
|
2278
|
+
this.domain = domain;
|
|
2279
|
+
this.authenticated = authenticated;
|
|
2280
|
+
this.principal = principal;
|
|
2281
|
+
this.claims = claims;
|
|
2595
2282
|
}
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
throw new Error("PipeIncrementalWriter already closed");
|
|
2599
|
-
this.writer._writeRecordBatch(batch);
|
|
2600
|
-
this.drain();
|
|
2283
|
+
static anonymous() {
|
|
2284
|
+
return new AuthContext("", false, null);
|
|
2601
2285
|
}
|
|
2602
|
-
|
|
2603
|
-
if (this.
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
const eos = new Uint8Array(new Int32Array([-1, 0]).buffer);
|
|
2607
|
-
this.writeFn(eos);
|
|
2286
|
+
requireAuthenticated() {
|
|
2287
|
+
if (!this.authenticated) {
|
|
2288
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2289
|
+
}
|
|
2608
2290
|
}
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2291
|
+
}
|
|
2292
|
+
// src/client/capabilities.ts
|
|
2293
|
+
var MAX_REQUEST_BYTES_HEADER = "VGI-Max-Request-Bytes";
|
|
2294
|
+
var UPLOAD_URL_HEADER = "VGI-Upload-URL-Support";
|
|
2295
|
+
var MAX_UPLOAD_BYTES_HEADER = "VGI-Max-Upload-Bytes";
|
|
2296
|
+
function parseHeaderInt(headers, name) {
|
|
2297
|
+
const raw = headers.get(name) ?? headers.get(name.toLowerCase());
|
|
2298
|
+
if (raw == null)
|
|
2299
|
+
return null;
|
|
2300
|
+
const parsed = Number.parseInt(raw, 10);
|
|
2301
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
2302
|
+
}
|
|
2303
|
+
function parseCapabilitiesFromHeaders(headers) {
|
|
2304
|
+
const uploadRaw = headers.get(UPLOAD_URL_HEADER) ?? headers.get(UPLOAD_URL_HEADER.toLowerCase());
|
|
2305
|
+
const uploadUrlSupport = uploadRaw === "true";
|
|
2306
|
+
let cacheExpiresAt = null;
|
|
2307
|
+
const cc = headers.get("Cache-Control") ?? headers.get("cache-control");
|
|
2308
|
+
if (cc) {
|
|
2309
|
+
for (const token of cc.split(",")) {
|
|
2310
|
+
const t = token.trim().toLowerCase();
|
|
2311
|
+
if (t.startsWith("max-age=")) {
|
|
2312
|
+
const seconds = Number.parseFloat(t.slice("max-age=".length));
|
|
2313
|
+
if (Number.isFinite(seconds)) {
|
|
2314
|
+
cacheExpiresAt = Date.now() + seconds * 1000;
|
|
2315
|
+
}
|
|
2316
|
+
break;
|
|
2317
|
+
}
|
|
2613
2318
|
}
|
|
2614
|
-
values.length = 0;
|
|
2615
2319
|
}
|
|
2320
|
+
return {
|
|
2321
|
+
maxRequestBytes: parseHeaderInt(headers, MAX_REQUEST_BYTES_HEADER),
|
|
2322
|
+
uploadUrlSupport,
|
|
2323
|
+
maxUploadBytes: parseHeaderInt(headers, MAX_UPLOAD_BYTES_HEADER),
|
|
2324
|
+
cacheExpiresAt
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
function isCapabilitySnapshotFresh(snapshot) {
|
|
2328
|
+
if (!snapshot)
|
|
2329
|
+
return false;
|
|
2330
|
+
if (snapshot.cacheExpiresAt == null)
|
|
2331
|
+
return true;
|
|
2332
|
+
return Date.now() < snapshot.cacheExpiresAt;
|
|
2616
2333
|
}
|
|
2617
2334
|
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2335
|
+
// src/client/stream.ts
|
|
2336
|
+
import { Field as Field2, makeData as makeData2, RecordBatch as RecordBatch2, Schema as Schema2, Struct as Struct2, vectorFromArray as vectorFromArray2 } from "@query-farm/apache-arrow";
|
|
2337
|
+
class HttpStreamSession {
|
|
2338
|
+
_baseUrl;
|
|
2339
|
+
_prefix;
|
|
2340
|
+
_method;
|
|
2341
|
+
_stateToken;
|
|
2342
|
+
_outputSchema;
|
|
2343
|
+
_inputSchema;
|
|
2621
2344
|
_onLog;
|
|
2345
|
+
_pendingBatches;
|
|
2346
|
+
_finished;
|
|
2622
2347
|
_header;
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
_outputSchema;
|
|
2628
|
-
_releaseBusy;
|
|
2629
|
-
_setDrainPromise;
|
|
2348
|
+
_compressionLevel;
|
|
2349
|
+
_compressFn;
|
|
2350
|
+
_decompressFn;
|
|
2351
|
+
_authorization;
|
|
2630
2352
|
_externalConfig;
|
|
2353
|
+
_postFn;
|
|
2631
2354
|
constructor(opts) {
|
|
2632
|
-
this.
|
|
2633
|
-
this.
|
|
2355
|
+
this._baseUrl = opts.baseUrl;
|
|
2356
|
+
this._prefix = opts.prefix;
|
|
2357
|
+
this._method = opts.method;
|
|
2358
|
+
this._stateToken = opts.stateToken;
|
|
2359
|
+
this._outputSchema = opts.outputSchema;
|
|
2360
|
+
this._inputSchema = opts.inputSchema;
|
|
2634
2361
|
this._onLog = opts.onLog;
|
|
2362
|
+
this._pendingBatches = opts.pendingBatches;
|
|
2363
|
+
this._finished = opts.finished;
|
|
2635
2364
|
this._header = opts.header;
|
|
2636
|
-
this.
|
|
2637
|
-
this.
|
|
2638
|
-
this.
|
|
2365
|
+
this._compressionLevel = opts.compressionLevel;
|
|
2366
|
+
this._compressFn = opts.compressFn;
|
|
2367
|
+
this._decompressFn = opts.decompressFn;
|
|
2368
|
+
this._authorization = opts.authorization;
|
|
2639
2369
|
this._externalConfig = opts.externalConfig;
|
|
2370
|
+
this._postFn = opts.postFn;
|
|
2371
|
+
}
|
|
2372
|
+
async _post(url, body) {
|
|
2373
|
+
if (this._postFn)
|
|
2374
|
+
return this._postFn(url, body);
|
|
2375
|
+
return fetch(url, {
|
|
2376
|
+
method: "POST",
|
|
2377
|
+
headers: this._buildHeaders(),
|
|
2378
|
+
body: await this._prepareBody(body)
|
|
2379
|
+
});
|
|
2640
2380
|
}
|
|
2641
2381
|
get header() {
|
|
2642
2382
|
return this._header;
|
|
2643
2383
|
}
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2384
|
+
_buildHeaders() {
|
|
2385
|
+
const headers = {
|
|
2386
|
+
"Content-Type": ARROW_CONTENT_TYPE
|
|
2387
|
+
};
|
|
2388
|
+
if (this._compressionLevel != null && this._compressFn) {
|
|
2389
|
+
headers["Content-Encoding"] = "zstd";
|
|
2390
|
+
}
|
|
2391
|
+
if (this._compressionLevel != null && this._decompressFn) {
|
|
2392
|
+
headers["Accept-Encoding"] = "zstd";
|
|
2393
|
+
}
|
|
2394
|
+
if (this._authorization) {
|
|
2395
|
+
headers.Authorization = this._authorization;
|
|
2396
|
+
}
|
|
2397
|
+
return headers;
|
|
2398
|
+
}
|
|
2399
|
+
async _prepareBody(content) {
|
|
2400
|
+
if (this._compressionLevel != null && this._compressFn) {
|
|
2401
|
+
return await this._compressFn(content, this._compressionLevel);
|
|
2402
|
+
}
|
|
2403
|
+
return content;
|
|
2404
|
+
}
|
|
2405
|
+
async _readResponse(resp) {
|
|
2406
|
+
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2407
|
+
if (resp.headers.get("Content-Encoding") === "zstd" && this._decompressFn) {
|
|
2408
|
+
body = new Uint8Array(await this._decompressFn(body));
|
|
2409
|
+
}
|
|
2410
|
+
return body;
|
|
2411
|
+
}
|
|
2412
|
+
async exchange(input) {
|
|
2413
|
+
if (this._stateToken === null) {
|
|
2414
|
+
throw new RpcError("ProtocolError", "Stream has finished — no state token available", "");
|
|
2415
|
+
}
|
|
2416
|
+
if (input.length === 0) {
|
|
2417
|
+
const zeroSchema = this._inputSchema ?? this._outputSchema;
|
|
2418
|
+
const emptyBatch = this._buildEmptyBatch(zeroSchema);
|
|
2419
|
+
const metadata2 = new Map;
|
|
2420
|
+
metadata2.set(STATE_KEY, this._stateToken);
|
|
2421
|
+
const batchWithMeta = new RecordBatch2(zeroSchema, emptyBatch.data, metadata2);
|
|
2422
|
+
return this._doExchange(zeroSchema, [batchWithMeta]);
|
|
2423
|
+
}
|
|
2424
|
+
const keys = Object.keys(input[0]);
|
|
2425
|
+
const fields = keys.map((key) => {
|
|
2426
|
+
let sample;
|
|
2427
|
+
for (const row of input) {
|
|
2428
|
+
if (row[key] != null) {
|
|
2429
|
+
sample = row[key];
|
|
2430
|
+
break;
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
const arrowType = inferArrowType(sample);
|
|
2434
|
+
const nullable = input.some((row) => row[key] == null);
|
|
2435
|
+
return new Field2(key, arrowType, nullable);
|
|
2436
|
+
});
|
|
2437
|
+
const inputSchema = new Schema2(fields);
|
|
2438
|
+
const children = inputSchema.fields.map((f) => {
|
|
2439
|
+
const values = input.map((row) => row[f.name]);
|
|
2440
|
+
return vectorFromArray2(values, f.type).data[0];
|
|
2441
|
+
});
|
|
2442
|
+
const structType = new Struct2(inputSchema.fields);
|
|
2443
|
+
const data = makeData2({
|
|
2444
|
+
type: structType,
|
|
2445
|
+
length: input.length,
|
|
2446
|
+
children,
|
|
2447
|
+
nullCount: 0
|
|
2448
|
+
});
|
|
2449
|
+
const metadata = new Map;
|
|
2450
|
+
metadata.set(STATE_KEY, this._stateToken);
|
|
2451
|
+
const batch = new RecordBatch2(inputSchema, data, metadata);
|
|
2452
|
+
return this._doExchange(inputSchema, [batch]);
|
|
2453
|
+
}
|
|
2454
|
+
async _doExchange(schema2, batches) {
|
|
2455
|
+
const body = serializeIpcStream(schema2, batches);
|
|
2456
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
2457
|
+
if (resp.status === 401) {
|
|
2458
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2459
|
+
}
|
|
2460
|
+
const responseBody = await this._readResponse(resp);
|
|
2461
|
+
const { batches: responseBatches } = await readResponseBatches(responseBody);
|
|
2462
|
+
let resultRows = [];
|
|
2463
|
+
for (const batch of responseBatches) {
|
|
2464
|
+
if (batch.numRows === 0) {
|
|
2465
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2466
|
+
const token2 = batch.metadata?.get(STATE_KEY);
|
|
2467
|
+
if (token2) {
|
|
2468
|
+
this._stateToken = token2;
|
|
2469
|
+
}
|
|
2470
|
+
continue;
|
|
2471
|
+
}
|
|
2472
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2473
|
+
if (token) {
|
|
2474
|
+
this._stateToken = token;
|
|
2475
|
+
}
|
|
2476
|
+
resultRows = extractBatchRows(batch);
|
|
2477
|
+
}
|
|
2478
|
+
return resultRows;
|
|
2479
|
+
}
|
|
2480
|
+
_buildEmptyBatch(schema2) {
|
|
2481
|
+
const children = schema2.fields.map((f) => {
|
|
2482
|
+
return makeData2({ type: f.type, length: 0, nullCount: 0 });
|
|
2483
|
+
});
|
|
2484
|
+
const structType = new Struct2(schema2.fields);
|
|
2485
|
+
const data = makeData2({
|
|
2486
|
+
type: structType,
|
|
2487
|
+
length: 0,
|
|
2488
|
+
children,
|
|
2489
|
+
nullCount: 0
|
|
2490
|
+
});
|
|
2491
|
+
return new RecordBatch2(schema2, data);
|
|
2492
|
+
}
|
|
2493
|
+
async* [Symbol.asyncIterator]() {
|
|
2494
|
+
for (let batch of this._pendingBatches) {
|
|
2649
2495
|
if (batch.numRows === 0) {
|
|
2650
2496
|
if (isExternalLocationBatch(batch)) {
|
|
2651
|
-
|
|
2652
|
-
}
|
|
2653
|
-
|
|
2497
|
+
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2498
|
+
} else {
|
|
2499
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2654
2500
|
continue;
|
|
2655
2501
|
}
|
|
2656
2502
|
}
|
|
2657
|
-
|
|
2503
|
+
yield extractBatchRows(batch);
|
|
2658
2504
|
}
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
if (this._outputStreamOpened)
|
|
2505
|
+
this._pendingBatches = [];
|
|
2506
|
+
if (this._finished)
|
|
2662
2507
|
return;
|
|
2663
|
-
this.
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2508
|
+
if (this._stateToken === null)
|
|
2509
|
+
return;
|
|
2510
|
+
while (true) {
|
|
2511
|
+
const stateToken = this._stateToken;
|
|
2512
|
+
if (stateToken === null)
|
|
2513
|
+
return;
|
|
2514
|
+
const responseBody = await this._sendContinuation(stateToken);
|
|
2515
|
+
const { batches } = await readResponseBatches(responseBody);
|
|
2516
|
+
let gotContinuation = false;
|
|
2517
|
+
for (let batch of batches) {
|
|
2518
|
+
if (batch.numRows === 0) {
|
|
2519
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2520
|
+
if (token) {
|
|
2521
|
+
this._stateToken = token;
|
|
2522
|
+
gotContinuation = true;
|
|
2523
|
+
continue;
|
|
2524
|
+
}
|
|
2525
|
+
if (isExternalLocationBatch(batch)) {
|
|
2526
|
+
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2527
|
+
} else {
|
|
2528
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2529
|
+
continue;
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
yield extractBatchRows(batch);
|
|
2533
|
+
}
|
|
2534
|
+
if (!gotContinuation)
|
|
2535
|
+
break;
|
|
2667
2536
|
}
|
|
2668
2537
|
}
|
|
2669
|
-
async
|
|
2670
|
-
|
|
2671
|
-
|
|
2538
|
+
async _sendContinuation(token) {
|
|
2539
|
+
const emptySchema = new Schema2([]);
|
|
2540
|
+
const metadata = new Map;
|
|
2541
|
+
metadata.set(STATE_KEY, token);
|
|
2542
|
+
const structType = new Struct2(emptySchema.fields);
|
|
2543
|
+
const data = makeData2({
|
|
2544
|
+
type: structType,
|
|
2545
|
+
length: 1,
|
|
2546
|
+
children: [],
|
|
2547
|
+
nullCount: 0
|
|
2548
|
+
});
|
|
2549
|
+
const batch = new RecordBatch2(emptySchema, data, metadata);
|
|
2550
|
+
const body = serializeIpcStream(emptySchema, [batch]);
|
|
2551
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
2552
|
+
if (resp.status === 401) {
|
|
2553
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2672
2554
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2555
|
+
return this._readResponse(resp);
|
|
2556
|
+
}
|
|
2557
|
+
close() {}
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
// src/client/uploadUrl.ts
|
|
2561
|
+
import { Field as Field3, Int64 as Int642, RecordBatchReader as RecordBatchReader4, Schema as Schema3 } from "@query-farm/apache-arrow";
|
|
2562
|
+
var UPLOAD_URL_METHOD = "__upload_url__";
|
|
2563
|
+
var UPLOAD_URL_PARAMS_SCHEMA = new Schema3([new Field3("count", new Int642, false)]);
|
|
2564
|
+
async function requestUploadUrls(baseUrl, prefix, count, authorization) {
|
|
2565
|
+
const body = buildRequestIpc(UPLOAD_URL_PARAMS_SCHEMA, { count: BigInt(count) }, UPLOAD_URL_METHOD);
|
|
2566
|
+
const headers = { "Content-Type": ARROW_CONTENT_TYPE };
|
|
2567
|
+
if (authorization)
|
|
2568
|
+
headers.Authorization = authorization;
|
|
2569
|
+
const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
|
|
2570
|
+
method: "POST",
|
|
2571
|
+
headers,
|
|
2572
|
+
body
|
|
2573
|
+
});
|
|
2574
|
+
if (resp.status === 404) {
|
|
2575
|
+
throw new RpcError("NotSupported", "Server does not support upload URLs", "");
|
|
2576
|
+
}
|
|
2577
|
+
if (resp.status === 401) {
|
|
2578
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2579
|
+
}
|
|
2580
|
+
if (!resp.ok) {
|
|
2581
|
+
throw new RpcError("HttpError", `__upload_url__/init failed: HTTP ${resp.status}`, "");
|
|
2582
|
+
}
|
|
2583
|
+
const respBody = new Uint8Array(await resp.arrayBuffer());
|
|
2584
|
+
const reader = await RecordBatchReader4.from(respBody);
|
|
2585
|
+
await reader.open();
|
|
2586
|
+
const pairs = [];
|
|
2587
|
+
for (const batch of reader.readAll()) {
|
|
2588
|
+
if (batch.numRows === 0)
|
|
2589
|
+
continue;
|
|
2590
|
+
for (let r = 0;r < batch.numRows; r++) {
|
|
2591
|
+
const uploadUrl = batch.getChildAt(0)?.get(r);
|
|
2592
|
+
const downloadUrl = batch.getChildAt(1)?.get(r);
|
|
2593
|
+
const expiresRaw = batch.getChildAt(2)?.get(r);
|
|
2594
|
+
let expiresAt;
|
|
2595
|
+
if (expiresRaw instanceof Date) {
|
|
2596
|
+
expiresAt = expiresRaw;
|
|
2597
|
+
} else if (typeof expiresRaw === "bigint") {
|
|
2598
|
+
expiresAt = new Date(Number(expiresRaw / 1000n));
|
|
2599
|
+
} else if (typeof expiresRaw === "number") {
|
|
2600
|
+
expiresAt = new Date(expiresRaw);
|
|
2707
2601
|
} else {
|
|
2708
|
-
|
|
2602
|
+
expiresAt = new Date;
|
|
2709
2603
|
}
|
|
2710
|
-
|
|
2711
|
-
const values = input.map((row) => row[f.name]);
|
|
2712
|
-
return vectorFromArray2(values, f.type).data[0];
|
|
2713
|
-
});
|
|
2714
|
-
const structType = new Struct2(inputSchema.fields);
|
|
2715
|
-
const data = makeData2({
|
|
2716
|
-
type: structType,
|
|
2717
|
-
length: input.length,
|
|
2718
|
-
children,
|
|
2719
|
-
nullCount: 0
|
|
2720
|
-
});
|
|
2721
|
-
batch = new RecordBatch2(inputSchema, data);
|
|
2604
|
+
pairs.push({ uploadUrl, downloadUrl, expiresAt });
|
|
2722
2605
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2606
|
+
}
|
|
2607
|
+
if (pairs.length === 0) {
|
|
2608
|
+
throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
|
|
2609
|
+
}
|
|
2610
|
+
return pairs;
|
|
2611
|
+
}
|
|
2612
|
+
async function buildPointerRequestBody(originalBody, downloadUrl) {
|
|
2613
|
+
const reader = await RecordBatchReader4.from(originalBody);
|
|
2614
|
+
await reader.open();
|
|
2615
|
+
const schema2 = reader.schema;
|
|
2616
|
+
if (!schema2) {
|
|
2617
|
+
throw new RpcError("ProtocolError", "Original request body has no schema", "");
|
|
2618
|
+
}
|
|
2619
|
+
const batches = reader.readAll();
|
|
2620
|
+
if (batches.length === 0) {
|
|
2621
|
+
throw new RpcError("ProtocolError", "Original request body has no batches", "");
|
|
2622
|
+
}
|
|
2623
|
+
const original = batches[0];
|
|
2624
|
+
const originalMeta = original.metadata ?? new Map;
|
|
2625
|
+
const pointer = makeExternalLocationBatch(schema2, downloadUrl);
|
|
2626
|
+
const merged = new Map(pointer.metadata ?? new Map);
|
|
2627
|
+
const method = originalMeta.get(RPC_METHOD_KEY);
|
|
2628
|
+
const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
|
|
2629
|
+
if (method)
|
|
2630
|
+
merged.set(RPC_METHOD_KEY, method);
|
|
2631
|
+
merged.set(REQUEST_VERSION_KEY, version);
|
|
2632
|
+
for (const [k, v] of originalMeta) {
|
|
2633
|
+
if (!merged.has(k))
|
|
2634
|
+
merged.set(k, v);
|
|
2635
|
+
}
|
|
2636
|
+
const { RecordBatch: RecordBatch3 } = await import("@query-farm/apache-arrow");
|
|
2637
|
+
const pointerWithMeta = new RecordBatch3(schema2, pointer.data, merged);
|
|
2638
|
+
return serializeIpcStream(schema2, [pointerWithMeta]);
|
|
2639
|
+
}
|
|
2640
|
+
async function externalizeRequestBody(body, opts) {
|
|
2641
|
+
const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
|
|
2642
|
+
const pair = pairs[0];
|
|
2643
|
+
if (opts.urlValidator) {
|
|
2644
|
+
opts.urlValidator(pair.uploadUrl);
|
|
2645
|
+
opts.urlValidator(pair.downloadUrl);
|
|
2646
|
+
}
|
|
2647
|
+
const putResp = await fetch(pair.uploadUrl, {
|
|
2648
|
+
method: "PUT",
|
|
2649
|
+
headers: { "Content-Type": ARROW_CONTENT_TYPE },
|
|
2650
|
+
body
|
|
2651
|
+
});
|
|
2652
|
+
if (!putResp.ok) {
|
|
2653
|
+
throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
|
|
2654
|
+
}
|
|
2655
|
+
return buildPointerRequestBody(body, pair.downloadUrl);
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
// src/client/connect.ts
|
|
2659
|
+
function httpConnect(baseUrl, options) {
|
|
2660
|
+
const prefix = (options?.prefix ?? "").replace(/\/+$/, "");
|
|
2661
|
+
const onLog = options?.onLog;
|
|
2662
|
+
const compressionLevel = options?.compressionLevel;
|
|
2663
|
+
const authorization = options?.authorization;
|
|
2664
|
+
const externalConfig = options?.externalLocation;
|
|
2665
|
+
let methodCache = null;
|
|
2666
|
+
let serverProtocolVersion = "";
|
|
2667
|
+
let compressFn;
|
|
2668
|
+
let decompressFn;
|
|
2669
|
+
let compressionLoaded = false;
|
|
2670
|
+
let capabilities = null;
|
|
2671
|
+
function updateCapabilitiesFromResponse(resp) {
|
|
2672
|
+
const next = parseCapabilitiesFromHeaders(resp.headers);
|
|
2673
|
+
if (next.maxRequestBytes != null || next.uploadUrlSupport) {
|
|
2674
|
+
capabilities = next;
|
|
2725
2675
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2676
|
+
}
|
|
2677
|
+
async function maybeExternalize(body) {
|
|
2678
|
+
const caps = isCapabilitySnapshotFresh(capabilities) ? capabilities : null;
|
|
2679
|
+
if (!caps)
|
|
2680
|
+
return body;
|
|
2681
|
+
if (!caps.uploadUrlSupport)
|
|
2682
|
+
return body;
|
|
2683
|
+
if (caps.maxRequestBytes == null || body.byteLength <= caps.maxRequestBytes)
|
|
2684
|
+
return body;
|
|
2685
|
+
return externalizeRequestBody(body, {
|
|
2686
|
+
baseUrl,
|
|
2687
|
+
prefix,
|
|
2688
|
+
authorization,
|
|
2689
|
+
urlValidator: externalConfig?.urlValidator ?? null
|
|
2690
|
+
});
|
|
2691
|
+
}
|
|
2692
|
+
async function postWithExternalization(url, body) {
|
|
2693
|
+
const sendBody = await maybeExternalize(body);
|
|
2694
|
+
let resp = await fetch(url, {
|
|
2695
|
+
method: "POST",
|
|
2696
|
+
headers: buildHeaders(),
|
|
2697
|
+
body: await prepareBody(sendBody)
|
|
2698
|
+
});
|
|
2699
|
+
updateCapabilitiesFromResponse(resp);
|
|
2700
|
+
if (resp.status === 413 && capabilities?.uploadUrlSupport && body.byteLength > 0) {
|
|
2701
|
+
const externalized = await externalizeRequestBody(body, {
|
|
2702
|
+
baseUrl,
|
|
2703
|
+
prefix,
|
|
2704
|
+
authorization,
|
|
2705
|
+
urlValidator: externalConfig?.urlValidator ?? null
|
|
2706
|
+
});
|
|
2707
|
+
resp = await fetch(url, {
|
|
2708
|
+
method: "POST",
|
|
2709
|
+
headers: buildHeaders(),
|
|
2710
|
+
body: await prepareBody(externalized)
|
|
2711
|
+
});
|
|
2712
|
+
updateCapabilitiesFromResponse(resp);
|
|
2737
2713
|
}
|
|
2714
|
+
return resp;
|
|
2738
2715
|
}
|
|
2739
|
-
async
|
|
2740
|
-
if (
|
|
2716
|
+
async function ensureCompression() {
|
|
2717
|
+
if (compressionLoaded || compressionLevel == null)
|
|
2741
2718
|
return;
|
|
2742
|
-
this._closed = true;
|
|
2743
|
-
if (this._inputWriter) {
|
|
2744
|
-
this._inputWriter.close();
|
|
2745
|
-
this._inputWriter = null;
|
|
2746
|
-
}
|
|
2747
2719
|
try {
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2720
|
+
const mod = await Promise.resolve().then(() => (init_zstd(), exports_zstd));
|
|
2721
|
+
compressFn = mod.zstdCompress;
|
|
2722
|
+
decompressFn = mod.zstdDecompress;
|
|
2751
2723
|
} catch {}
|
|
2752
|
-
|
|
2724
|
+
compressionLoaded = true;
|
|
2753
2725
|
}
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
const structType = new Struct2(tickSchema.fields);
|
|
2761
|
-
const tickData = makeData2({
|
|
2762
|
-
type: structType,
|
|
2763
|
-
length: 0,
|
|
2764
|
-
children: [],
|
|
2765
|
-
nullCount: 0
|
|
2766
|
-
});
|
|
2767
|
-
const tickBatch = new RecordBatch2(tickSchema, tickData);
|
|
2768
|
-
while (true) {
|
|
2769
|
-
this._inputWriter.write(tickBatch);
|
|
2770
|
-
await this._ensureOutputStream();
|
|
2771
|
-
const outputBatch = await this._readOutputBatch();
|
|
2772
|
-
if (outputBatch === null) {
|
|
2773
|
-
break;
|
|
2774
|
-
}
|
|
2775
|
-
yield extractBatchRows(outputBatch);
|
|
2776
|
-
}
|
|
2777
|
-
} finally {
|
|
2778
|
-
if (this._inputWriter) {
|
|
2779
|
-
this._inputWriter.close();
|
|
2780
|
-
this._inputWriter = null;
|
|
2781
|
-
}
|
|
2782
|
-
try {
|
|
2783
|
-
if (this._outputStreamOpened) {
|
|
2784
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2785
|
-
}
|
|
2786
|
-
} catch {}
|
|
2787
|
-
this._closed = true;
|
|
2788
|
-
this._releaseBusy();
|
|
2726
|
+
function buildHeaders() {
|
|
2727
|
+
const headers = {
|
|
2728
|
+
"Content-Type": ARROW_CONTENT_TYPE
|
|
2729
|
+
};
|
|
2730
|
+
if (compressionLevel != null && compressFn) {
|
|
2731
|
+
headers["Content-Encoding"] = "zstd";
|
|
2789
2732
|
}
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
if (this._closed)
|
|
2793
|
-
return;
|
|
2794
|
-
this._closed = true;
|
|
2795
|
-
if (this._inputWriter) {
|
|
2796
|
-
this._inputWriter.close();
|
|
2797
|
-
this._inputWriter = null;
|
|
2798
|
-
} else {
|
|
2799
|
-
const emptySchema = new Schema3([]);
|
|
2800
|
-
const ipc = serializeIpcStream(emptySchema, []);
|
|
2801
|
-
this._writeFn(ipc);
|
|
2733
|
+
if (compressionLevel != null && decompressFn) {
|
|
2734
|
+
headers["Accept-Encoding"] = "zstd";
|
|
2802
2735
|
}
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
if (!this._outputStreamOpened) {
|
|
2806
|
-
const schema2 = await this._reader.openNextStream();
|
|
2807
|
-
if (schema2) {
|
|
2808
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2809
|
-
}
|
|
2810
|
-
} else {
|
|
2811
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2812
|
-
}
|
|
2813
|
-
} catch {} finally {
|
|
2814
|
-
this._releaseBusy();
|
|
2815
|
-
}
|
|
2816
|
-
})();
|
|
2817
|
-
this._setDrainPromise(drainPromise);
|
|
2818
|
-
}
|
|
2819
|
-
}
|
|
2820
|
-
function pipeConnect(readable, writable, options) {
|
|
2821
|
-
const onLog = options?.onLog;
|
|
2822
|
-
const externalConfig = options?.externalLocation;
|
|
2823
|
-
let reader = null;
|
|
2824
|
-
let readerPromise = null;
|
|
2825
|
-
let methodCache = null;
|
|
2826
|
-
let protocolName = "";
|
|
2827
|
-
let serverProtocolVersion = "";
|
|
2828
|
-
let _busy = false;
|
|
2829
|
-
let _drainPromise = null;
|
|
2830
|
-
let closed = false;
|
|
2831
|
-
const writeFn = (bytes) => {
|
|
2832
|
-
writable.write(bytes);
|
|
2833
|
-
writable.flush?.();
|
|
2834
|
-
};
|
|
2835
|
-
async function ensureReader() {
|
|
2836
|
-
if (reader)
|
|
2837
|
-
return reader;
|
|
2838
|
-
if (!readerPromise) {
|
|
2839
|
-
readerPromise = IpcStreamReader.create(readable);
|
|
2736
|
+
if (authorization) {
|
|
2737
|
+
headers.Authorization = authorization;
|
|
2840
2738
|
}
|
|
2841
|
-
|
|
2842
|
-
return reader;
|
|
2739
|
+
return headers;
|
|
2843
2740
|
}
|
|
2844
|
-
async function
|
|
2845
|
-
if (
|
|
2846
|
-
await
|
|
2847
|
-
_drainPromise = null;
|
|
2848
|
-
}
|
|
2849
|
-
if (_busy) {
|
|
2850
|
-
throw new Error("Pipe transport is busy — another call or stream is in progress. " + "Pipe connections are single-threaded; wait for the current operation to complete.");
|
|
2741
|
+
async function prepareBody(content) {
|
|
2742
|
+
if (compressionLevel != null && compressFn) {
|
|
2743
|
+
return await compressFn(content, compressionLevel);
|
|
2851
2744
|
}
|
|
2852
|
-
|
|
2745
|
+
return content;
|
|
2853
2746
|
}
|
|
2854
|
-
function
|
|
2855
|
-
|
|
2747
|
+
function checkAuth(resp) {
|
|
2748
|
+
if (resp.status === 401) {
|
|
2749
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2750
|
+
}
|
|
2856
2751
|
}
|
|
2857
|
-
function
|
|
2858
|
-
|
|
2752
|
+
async function readResponse(resp) {
|
|
2753
|
+
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2754
|
+
if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
|
|
2755
|
+
body = new Uint8Array(await decompressFn(body));
|
|
2756
|
+
}
|
|
2757
|
+
return body;
|
|
2859
2758
|
}
|
|
2860
2759
|
async function ensureMethodCache() {
|
|
2861
2760
|
if (methodCache)
|
|
2862
2761
|
return methodCache;
|
|
2863
|
-
await
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
protocolName = desc.protocolName;
|
|
2875
|
-
serverProtocolVersion = desc.protocolVersion;
|
|
2876
|
-
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2877
|
-
return methodCache;
|
|
2878
|
-
} finally {
|
|
2879
|
-
releaseBusy();
|
|
2880
|
-
}
|
|
2762
|
+
await ensureCompression();
|
|
2763
|
+
const desc = await httpIntrospect(baseUrl, {
|
|
2764
|
+
prefix,
|
|
2765
|
+
authorization,
|
|
2766
|
+
compressionLevel,
|
|
2767
|
+
compressFn,
|
|
2768
|
+
decompressFn
|
|
2769
|
+
});
|
|
2770
|
+
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2771
|
+
serverProtocolVersion = desc.protocolVersion;
|
|
2772
|
+
return methodCache;
|
|
2881
2773
|
}
|
|
2882
2774
|
return {
|
|
2883
2775
|
async call(method, params) {
|
|
2776
|
+
await ensureCompression();
|
|
2884
2777
|
const methods = await ensureMethodCache();
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2778
|
+
const info = methods.get(method);
|
|
2779
|
+
if (!info) {
|
|
2780
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2781
|
+
}
|
|
2782
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2783
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2784
|
+
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}`, body);
|
|
2785
|
+
checkAuth(resp);
|
|
2786
|
+
const responseBody = await readResponse(resp);
|
|
2787
|
+
const { batches } = await readResponseBatches(responseBody);
|
|
2788
|
+
let resultBatch = null;
|
|
2789
|
+
for (let batch of batches) {
|
|
2790
|
+
if (batch.numRows === 0) {
|
|
2791
|
+
if (isExternalLocationBatch(batch)) {
|
|
2792
|
+
batch = await resolveExternalLocation(batch, externalConfig);
|
|
2793
|
+
} else {
|
|
2794
|
+
dispatchLogOrError(batch, onLog);
|
|
2795
|
+
continue;
|
|
2796
|
+
}
|
|
2890
2797
|
}
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2798
|
+
resultBatch = batch;
|
|
2799
|
+
}
|
|
2800
|
+
if (!resultBatch) {
|
|
2801
|
+
return null;
|
|
2802
|
+
}
|
|
2803
|
+
const rows = extractBatchRows(resultBatch);
|
|
2804
|
+
if (rows.length === 0)
|
|
2805
|
+
return null;
|
|
2806
|
+
const result = rows[0];
|
|
2807
|
+
if (info.resultSchema.fields.length === 0)
|
|
2808
|
+
return null;
|
|
2809
|
+
return result;
|
|
2810
|
+
},
|
|
2811
|
+
async stream(method, params) {
|
|
2812
|
+
await ensureCompression();
|
|
2813
|
+
const methods = await ensureMethodCache();
|
|
2814
|
+
const info = methods.get(method);
|
|
2815
|
+
if (!info) {
|
|
2816
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2817
|
+
}
|
|
2818
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2819
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2820
|
+
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}/init`, body);
|
|
2821
|
+
checkAuth(resp);
|
|
2822
|
+
const responseBody = await readResponse(resp);
|
|
2823
|
+
let header = null;
|
|
2824
|
+
let stateToken = null;
|
|
2825
|
+
const pendingBatches = [];
|
|
2826
|
+
let finished = false;
|
|
2827
|
+
let streamSchema = null;
|
|
2828
|
+
if (info.headerSchema) {
|
|
2829
|
+
const reader = await readSequentialStreams(responseBody);
|
|
2830
|
+
const headerStream = await reader.readStream();
|
|
2831
|
+
if (headerStream) {
|
|
2832
|
+
for (const batch of headerStream.batches) {
|
|
2833
|
+
if (batch.numRows === 0) {
|
|
2834
|
+
dispatchLogOrError(batch, onLog);
|
|
2835
|
+
continue;
|
|
2836
|
+
}
|
|
2837
|
+
const rows = extractBatchRows(batch);
|
|
2838
|
+
if (rows.length > 0) {
|
|
2839
|
+
header = rows[0];
|
|
2840
|
+
}
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
const dataStream = await reader.readStream();
|
|
2844
|
+
if (dataStream) {
|
|
2845
|
+
streamSchema = dataStream.schema;
|
|
2898
2846
|
}
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
if (
|
|
2903
|
-
|
|
2904
|
-
|
|
2847
|
+
const headerErrorBatches = [];
|
|
2848
|
+
if (dataStream) {
|
|
2849
|
+
for (const batch of dataStream.batches) {
|
|
2850
|
+
if (batch.numRows === 0) {
|
|
2851
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2852
|
+
if (token) {
|
|
2853
|
+
stateToken = token;
|
|
2854
|
+
continue;
|
|
2855
|
+
}
|
|
2856
|
+
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2857
|
+
if (level === "EXCEPTION") {
|
|
2858
|
+
headerErrorBatches.push(batch);
|
|
2859
|
+
continue;
|
|
2860
|
+
}
|
|
2905
2861
|
dispatchLogOrError(batch, onLog);
|
|
2906
2862
|
continue;
|
|
2907
2863
|
}
|
|
2864
|
+
pendingBatches.push(batch);
|
|
2908
2865
|
}
|
|
2909
|
-
resultBatch = batch;
|
|
2910
2866
|
}
|
|
2911
|
-
if (
|
|
2912
|
-
|
|
2867
|
+
if (headerErrorBatches.length > 0) {
|
|
2868
|
+
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2869
|
+
pendingBatches.push(...headerErrorBatches);
|
|
2870
|
+
} else {
|
|
2871
|
+
for (const batch of headerErrorBatches) {
|
|
2872
|
+
dispatchLogOrError(batch, onLog);
|
|
2873
|
+
}
|
|
2874
|
+
}
|
|
2913
2875
|
}
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
return null;
|
|
2917
|
-
if (info.resultSchema.fields.length === 0)
|
|
2918
|
-
return null;
|
|
2919
|
-
return rows[0];
|
|
2920
|
-
} finally {
|
|
2921
|
-
releaseBusy();
|
|
2922
|
-
}
|
|
2923
|
-
},
|
|
2924
|
-
async stream(method, params) {
|
|
2925
|
-
const methods = await ensureMethodCache();
|
|
2926
|
-
await acquireBusy();
|
|
2927
|
-
try {
|
|
2928
|
-
const info = methods.get(method);
|
|
2929
|
-
if (!info) {
|
|
2930
|
-
throw new Error(`Unknown method: '${method}'`);
|
|
2876
|
+
if (!dataStream && !stateToken) {
|
|
2877
|
+
finished = true;
|
|
2931
2878
|
}
|
|
2932
|
-
|
|
2933
|
-
const
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
dispatchLogOrError(batch, onLog);
|
|
2943
|
-
continue;
|
|
2944
|
-
}
|
|
2945
|
-
const rows = extractBatchRows(batch);
|
|
2946
|
-
if (rows.length > 0) {
|
|
2947
|
-
header = rows[0];
|
|
2948
|
-
}
|
|
2879
|
+
} else {
|
|
2880
|
+
const { schema: responseSchema, batches } = await readResponseBatches(responseBody);
|
|
2881
|
+
streamSchema = responseSchema;
|
|
2882
|
+
const errorBatches = [];
|
|
2883
|
+
for (const batch of batches) {
|
|
2884
|
+
if (batch.numRows === 0) {
|
|
2885
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2886
|
+
if (token) {
|
|
2887
|
+
stateToken = token;
|
|
2888
|
+
continue;
|
|
2949
2889
|
}
|
|
2890
|
+
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2891
|
+
if (level === "EXCEPTION") {
|
|
2892
|
+
errorBatches.push(batch);
|
|
2893
|
+
continue;
|
|
2894
|
+
}
|
|
2895
|
+
dispatchLogOrError(batch, onLog);
|
|
2896
|
+
continue;
|
|
2950
2897
|
}
|
|
2898
|
+
pendingBatches.push(batch);
|
|
2951
2899
|
}
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
try {
|
|
2965
|
-
const r = await ensureReader();
|
|
2966
|
-
const emptySchema = new Schema3([]);
|
|
2967
|
-
const ipc = serializeIpcStream(emptySchema, []);
|
|
2968
|
-
writeFn(ipc);
|
|
2969
|
-
const outStream = await r.readStream();
|
|
2970
|
-
} catch {}
|
|
2971
|
-
releaseBusy();
|
|
2972
|
-
throw e;
|
|
2900
|
+
if (errorBatches.length > 0) {
|
|
2901
|
+
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2902
|
+
pendingBatches.push(...errorBatches);
|
|
2903
|
+
} else {
|
|
2904
|
+
for (const batch of errorBatches) {
|
|
2905
|
+
dispatchLogOrError(batch, onLog);
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
}
|
|
2909
|
+
}
|
|
2910
|
+
if (pendingBatches.length === 0 && stateToken === null) {
|
|
2911
|
+
finished = true;
|
|
2973
2912
|
}
|
|
2913
|
+
const outputSchema = (streamSchema && streamSchema.fields.length > 0 ? streamSchema : null) ?? (pendingBatches.length > 0 ? pendingBatches[0].schema : null) ?? info.outputSchema ?? info.resultSchema;
|
|
2914
|
+
return new HttpStreamSession({
|
|
2915
|
+
baseUrl,
|
|
2916
|
+
prefix,
|
|
2917
|
+
method,
|
|
2918
|
+
stateToken,
|
|
2919
|
+
outputSchema,
|
|
2920
|
+
inputSchema: info.inputSchema,
|
|
2921
|
+
onLog,
|
|
2922
|
+
pendingBatches,
|
|
2923
|
+
finished,
|
|
2924
|
+
header,
|
|
2925
|
+
compressionLevel,
|
|
2926
|
+
compressFn,
|
|
2927
|
+
decompressFn,
|
|
2928
|
+
authorization,
|
|
2929
|
+
externalConfig,
|
|
2930
|
+
postFn: postWithExternalization
|
|
2931
|
+
});
|
|
2974
2932
|
},
|
|
2975
2933
|
async describe() {
|
|
2976
|
-
|
|
2977
|
-
return {
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2934
|
+
await ensureCompression();
|
|
2935
|
+
return httpIntrospect(baseUrl, {
|
|
2936
|
+
prefix,
|
|
2937
|
+
authorization,
|
|
2938
|
+
compressionLevel,
|
|
2939
|
+
compressFn,
|
|
2940
|
+
decompressFn
|
|
2941
|
+
});
|
|
2982
2942
|
},
|
|
2983
|
-
close() {
|
|
2984
|
-
if (closed)
|
|
2985
|
-
return;
|
|
2986
|
-
closed = true;
|
|
2987
|
-
writable.end();
|
|
2988
|
-
}
|
|
2943
|
+
close() {}
|
|
2989
2944
|
};
|
|
2990
2945
|
}
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
2946
|
+
// src/client/oauth.ts
|
|
2947
|
+
function parseMetadataJson(json) {
|
|
2948
|
+
const result = {
|
|
2949
|
+
resource: json.resource,
|
|
2950
|
+
authorizationServers: json.authorization_servers
|
|
2951
|
+
};
|
|
2952
|
+
if (json.scopes_supported)
|
|
2953
|
+
result.scopesSupported = json.scopes_supported;
|
|
2954
|
+
if (json.bearer_methods_supported)
|
|
2955
|
+
result.bearerMethodsSupported = json.bearer_methods_supported;
|
|
2956
|
+
if (json.resource_signing_alg_values_supported)
|
|
2957
|
+
result.resourceSigningAlgValuesSupported = json.resource_signing_alg_values_supported;
|
|
2958
|
+
if (json.resource_name)
|
|
2959
|
+
result.resourceName = json.resource_name;
|
|
2960
|
+
if (json.resource_documentation)
|
|
2961
|
+
result.resourceDocumentation = json.resource_documentation;
|
|
2962
|
+
if (json.resource_policy_uri)
|
|
2963
|
+
result.resourcePolicyUri = json.resource_policy_uri;
|
|
2964
|
+
if (json.resource_tos_uri)
|
|
2965
|
+
result.resourceTosUri = json.resource_tos_uri;
|
|
2966
|
+
if (json.client_id)
|
|
2967
|
+
result.clientId = json.client_id;
|
|
2968
|
+
if (json.client_secret)
|
|
2969
|
+
result.clientSecret = json.client_secret;
|
|
2970
|
+
if (json.use_id_token_as_bearer)
|
|
2971
|
+
result.useIdTokenAsBearer = json.use_id_token_as_bearer;
|
|
2972
|
+
if (json.device_code_client_id)
|
|
2973
|
+
result.deviceCodeClientId = json.device_code_client_id;
|
|
2974
|
+
if (json.device_code_client_secret)
|
|
2975
|
+
result.deviceCodeClientSecret = json.device_code_client_secret;
|
|
2976
|
+
return result;
|
|
2977
|
+
}
|
|
2978
|
+
async function httpOAuthMetadata(baseUrl, prefix) {
|
|
2979
|
+
const effectivePrefix = (prefix ?? "").replace(/\/+$/, "");
|
|
2980
|
+
const metadataUrl = `${baseUrl.replace(/\/+$/, "")}/.well-known/oauth-protected-resource${effectivePrefix}`;
|
|
2981
|
+
try {
|
|
2982
|
+
return await fetchOAuthMetadata(metadataUrl);
|
|
2983
|
+
} catch {
|
|
2984
|
+
return null;
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
async function fetchOAuthMetadata(metadataUrl) {
|
|
2988
|
+
const response = await fetch(metadataUrl);
|
|
2989
|
+
if (!response.ok) {
|
|
2990
|
+
throw new Error(`Failed to fetch OAuth metadata from ${metadataUrl}: ${response.status}`);
|
|
2991
|
+
}
|
|
2992
|
+
const json = await response.json();
|
|
2993
|
+
return parseMetadataJson(json);
|
|
2994
|
+
}
|
|
2995
|
+
function parseResourceMetadataUrl(wwwAuthenticate) {
|
|
2996
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
2997
|
+
if (!bearerMatch)
|
|
2998
|
+
return null;
|
|
2999
|
+
const params = bearerMatch[1];
|
|
3000
|
+
const metadataMatch = params.match(/resource_metadata="([^"]+)"/);
|
|
3001
|
+
if (!metadataMatch)
|
|
3002
|
+
return null;
|
|
3003
|
+
return metadataMatch[1];
|
|
3004
|
+
}
|
|
3005
|
+
function parseClientId(wwwAuthenticate) {
|
|
3006
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3007
|
+
if (!bearerMatch)
|
|
3008
|
+
return null;
|
|
3009
|
+
const params = bearerMatch[1];
|
|
3010
|
+
const clientIdMatch = params.match(/client_id="([^"]+)"/);
|
|
3011
|
+
if (!clientIdMatch)
|
|
3012
|
+
return null;
|
|
3013
|
+
return clientIdMatch[1];
|
|
3014
|
+
}
|
|
3015
|
+
function parseClientSecret(wwwAuthenticate) {
|
|
3016
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3017
|
+
if (!bearerMatch)
|
|
3018
|
+
return null;
|
|
3019
|
+
const params = bearerMatch[1];
|
|
3020
|
+
const match = params.match(/client_secret="([^"]+)"/);
|
|
3021
|
+
if (!match)
|
|
3022
|
+
return null;
|
|
3023
|
+
return match[1];
|
|
3024
|
+
}
|
|
3025
|
+
function parseUseIdTokenAsBearer(wwwAuthenticate) {
|
|
3026
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3027
|
+
if (!bearerMatch)
|
|
3028
|
+
return false;
|
|
3029
|
+
const params = bearerMatch[1];
|
|
3030
|
+
const match = params.match(/use_id_token_as_bearer="([^"]+)"/);
|
|
3031
|
+
if (!match)
|
|
3032
|
+
return false;
|
|
3033
|
+
return match[1] === "true";
|
|
3034
|
+
}
|
|
3035
|
+
function parseDeviceCodeClientId(wwwAuthenticate) {
|
|
3036
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3037
|
+
if (!bearerMatch)
|
|
3038
|
+
return null;
|
|
3039
|
+
const params = bearerMatch[1];
|
|
3040
|
+
const match = params.match(/device_code_client_id="([^"]+)"/);
|
|
3041
|
+
if (!match)
|
|
3042
|
+
return null;
|
|
3043
|
+
return match[1];
|
|
3044
|
+
}
|
|
3045
|
+
function parseDeviceCodeClientSecret(wwwAuthenticate) {
|
|
3046
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3047
|
+
if (!bearerMatch)
|
|
3048
|
+
return null;
|
|
3049
|
+
const params = bearerMatch[1];
|
|
3050
|
+
const match = params.match(/device_code_client_secret="([^"]+)"/);
|
|
3051
|
+
if (!match)
|
|
3052
|
+
return null;
|
|
3053
|
+
return match[1];
|
|
3023
3054
|
}
|
|
3024
3055
|
// src/http/auth.ts
|
|
3025
3056
|
function oauthResourceMetadataToJson(metadata) {
|
|
@@ -3304,6 +3335,7 @@ var TransportKind;
|
|
|
3304
3335
|
TransportKind2["PIPE"] = "pipe";
|
|
3305
3336
|
TransportKind2["HTTP"] = "http";
|
|
3306
3337
|
TransportKind2["UNIX"] = "unix";
|
|
3338
|
+
TransportKind2["TCP"] = "tcp";
|
|
3307
3339
|
})(TransportKind ||= {});
|
|
3308
3340
|
var EMPTY_COOKIES = new Map;
|
|
3309
3341
|
function cookieNotUnaryHttpError() {
|
|
@@ -9236,11 +9268,236 @@ function onceExit(proc) {
|
|
|
9236
9268
|
function delay(ms) {
|
|
9237
9269
|
return new Promise((r) => setTimeout(r, Math.max(0, ms)));
|
|
9238
9270
|
}
|
|
9271
|
+
// src/launcher/serve-tcp.ts
|
|
9272
|
+
import { createServer } from "node:net";
|
|
9273
|
+
var EMPTY_SCHEMA6 = schema([]);
|
|
9274
|
+
async function serveTcp(protocol, options = {}) {
|
|
9275
|
+
const host = options.host ?? "127.0.0.1";
|
|
9276
|
+
const requestedPort = options.port ?? 0;
|
|
9277
|
+
const idleTimeoutS = options.idleTimeout ?? 300;
|
|
9278
|
+
const startupGraceS = options.startupGraceSeconds ?? 5;
|
|
9279
|
+
const protocolVersion = options.protocolVersion ?? "";
|
|
9280
|
+
const serverId = options.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
9281
|
+
const enableDescribe = options.enableDescribe ?? true;
|
|
9282
|
+
const dispatchHook = options.dispatchHook ?? null;
|
|
9283
|
+
const externalConfig = options.externalLocation;
|
|
9284
|
+
const onServeStart = options.onServeStart ?? null;
|
|
9285
|
+
const backlog = options.backlog ?? 128;
|
|
9286
|
+
const announcementSink = options.announcementSink ?? process.stdout;
|
|
9287
|
+
let describePromise = null;
|
|
9288
|
+
function describeInfo() {
|
|
9289
|
+
if (!describePromise) {
|
|
9290
|
+
describePromise = buildDescribeBatch(protocol.name, protocol.getMethods(), serverId).then(({ batch, metadata }) => ({
|
|
9291
|
+
batch,
|
|
9292
|
+
protocolHash: metadata.get("vgi_rpc.protocol_hash") ?? ""
|
|
9293
|
+
}));
|
|
9294
|
+
}
|
|
9295
|
+
return describePromise;
|
|
9296
|
+
}
|
|
9297
|
+
let serveStartFired = false;
|
|
9298
|
+
async function notifyTransport() {
|
|
9299
|
+
if (serveStartFired)
|
|
9300
|
+
return;
|
|
9301
|
+
if (onServeStart) {
|
|
9302
|
+
await onServeStart("tcp" /* TCP */);
|
|
9303
|
+
}
|
|
9304
|
+
serveStartFired = true;
|
|
9305
|
+
}
|
|
9306
|
+
const server = createServer({ allowHalfOpen: false });
|
|
9307
|
+
let activeConnections = 0;
|
|
9308
|
+
let idleTimer = null;
|
|
9309
|
+
let resolveDone = () => {};
|
|
9310
|
+
let rejectDone = () => {};
|
|
9311
|
+
const done = new Promise((resolve, reject) => {
|
|
9312
|
+
resolveDone = resolve;
|
|
9313
|
+
rejectDone = reject;
|
|
9314
|
+
});
|
|
9315
|
+
let stopped = false;
|
|
9316
|
+
function armIdleTimer() {
|
|
9317
|
+
if (idleTimeoutS <= 0)
|
|
9318
|
+
return;
|
|
9319
|
+
if (idleTimer)
|
|
9320
|
+
clearTimeout(idleTimer);
|
|
9321
|
+
idleTimer = setTimeout(() => {
|
|
9322
|
+
if (activeConnections === 0 && !stopped) {
|
|
9323
|
+
shutdown();
|
|
9324
|
+
}
|
|
9325
|
+
}, idleTimeoutS * 1000);
|
|
9326
|
+
}
|
|
9327
|
+
function disarmIdleTimer() {
|
|
9328
|
+
if (idleTimer) {
|
|
9329
|
+
clearTimeout(idleTimer);
|
|
9330
|
+
idleTimer = null;
|
|
9331
|
+
}
|
|
9332
|
+
}
|
|
9333
|
+
async function shutdown() {
|
|
9334
|
+
if (stopped)
|
|
9335
|
+
return;
|
|
9336
|
+
stopped = true;
|
|
9337
|
+
disarmIdleTimer();
|
|
9338
|
+
await new Promise((resolve) => {
|
|
9339
|
+
server.close(() => resolve());
|
|
9340
|
+
});
|
|
9341
|
+
resolveDone();
|
|
9342
|
+
}
|
|
9343
|
+
server.on("connection", (socket) => {
|
|
9344
|
+
try {
|
|
9345
|
+
socket.setNoDelay(true);
|
|
9346
|
+
} catch {}
|
|
9347
|
+
activeConnections += 1;
|
|
9348
|
+
disarmIdleTimer();
|
|
9349
|
+
handleConnection(socket).catch((err2) => {
|
|
9350
|
+
process.stderr.write(`vgi-rpc/tcp: connection failed: ${err2?.message ?? err2}
|
|
9351
|
+
`);
|
|
9352
|
+
}).finally(() => {
|
|
9353
|
+
activeConnections -= 1;
|
|
9354
|
+
socket.destroy();
|
|
9355
|
+
if (activeConnections === 0 && !stopped) {
|
|
9356
|
+
armIdleTimer();
|
|
9357
|
+
}
|
|
9358
|
+
});
|
|
9359
|
+
});
|
|
9360
|
+
server.on("error", (err2) => {
|
|
9361
|
+
if (stopped)
|
|
9362
|
+
return;
|
|
9363
|
+
rejectDone(err2);
|
|
9364
|
+
});
|
|
9365
|
+
async function handleConnection(socket) {
|
|
9366
|
+
const reader = await IpcStreamReader.create(socket);
|
|
9367
|
+
const writer = new IpcStreamWriter(socket);
|
|
9368
|
+
try {
|
|
9369
|
+
await notifyTransport();
|
|
9370
|
+
while (true) {
|
|
9371
|
+
try {
|
|
9372
|
+
await serveOnce(reader, writer);
|
|
9373
|
+
} catch (e) {
|
|
9374
|
+
const err2 = e;
|
|
9375
|
+
if (err2?.message?.includes("closed") || err2?.message?.includes("Expected Schema Message") || err2?.message?.includes("null or length 0") || err2?.message?.includes("EOF") || err2?.code === "EPIPE" || err2?.code === "ERR_STREAM_PREMATURE_CLOSE" || err2?.code === "ERR_STREAM_DESTROYED") {
|
|
9376
|
+
return;
|
|
9377
|
+
}
|
|
9378
|
+
throw e;
|
|
9379
|
+
}
|
|
9380
|
+
}
|
|
9381
|
+
} finally {
|
|
9382
|
+
try {
|
|
9383
|
+
await reader.cancel();
|
|
9384
|
+
} catch {}
|
|
9385
|
+
}
|
|
9386
|
+
}
|
|
9387
|
+
async function serveOnce(reader, writer) {
|
|
9388
|
+
const stream = await reader.readStream();
|
|
9389
|
+
if (!stream) {
|
|
9390
|
+
throw new Error("EOF");
|
|
9391
|
+
}
|
|
9392
|
+
const { schema: schema2, batches } = stream;
|
|
9393
|
+
if (batches.length === 0) {
|
|
9394
|
+
const err2 = new RpcError("ProtocolError", "Request stream contains no batches", "");
|
|
9395
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, err2, serverId, null);
|
|
9396
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9397
|
+
return;
|
|
9398
|
+
}
|
|
9399
|
+
const batch = batches[0];
|
|
9400
|
+
let methodName;
|
|
9401
|
+
let params;
|
|
9402
|
+
let requestId;
|
|
9403
|
+
try {
|
|
9404
|
+
const parsed = parseRequest(schema2, batch);
|
|
9405
|
+
methodName = parsed.methodName;
|
|
9406
|
+
params = parsed.params;
|
|
9407
|
+
requestId = parsed.requestId;
|
|
9408
|
+
} catch (e) {
|
|
9409
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, e, serverId, null);
|
|
9410
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9411
|
+
if (e instanceof VersionError || e instanceof RpcError)
|
|
9412
|
+
return;
|
|
9413
|
+
throw e;
|
|
9414
|
+
}
|
|
9415
|
+
if (methodName === DESCRIBE_METHOD_NAME && enableDescribe) {
|
|
9416
|
+
const { batch: descBatch } = await describeInfo();
|
|
9417
|
+
await writer.writeStream(descBatch.schema, [descBatch]);
|
|
9418
|
+
return;
|
|
9419
|
+
}
|
|
9420
|
+
const methods = protocol.getMethods();
|
|
9421
|
+
const method = methods.get(methodName);
|
|
9422
|
+
if (!method) {
|
|
9423
|
+
const available = [...methods.keys()].sort();
|
|
9424
|
+
const err2 = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
9425
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, err2, serverId, requestId);
|
|
9426
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9427
|
+
return;
|
|
9428
|
+
}
|
|
9429
|
+
const methodType = method.type === "unary" /* UNARY */ ? "unary" : "stream";
|
|
9430
|
+
let requestData;
|
|
9431
|
+
try {
|
|
9432
|
+
requestData = serializeBatch(batch);
|
|
9433
|
+
} catch {}
|
|
9434
|
+
const { protocolHash } = await describeInfo();
|
|
9435
|
+
const info = {
|
|
9436
|
+
method: methodName,
|
|
9437
|
+
methodType,
|
|
9438
|
+
serverId,
|
|
9439
|
+
requestId,
|
|
9440
|
+
protocol: protocol.name,
|
|
9441
|
+
protocolHash,
|
|
9442
|
+
protocolVersion,
|
|
9443
|
+
kind: "tcp" /* TCP */,
|
|
9444
|
+
principal: "",
|
|
9445
|
+
authDomain: "",
|
|
9446
|
+
authenticated: false,
|
|
9447
|
+
remoteAddr: "",
|
|
9448
|
+
requestData
|
|
9449
|
+
};
|
|
9450
|
+
const stats = {
|
|
9451
|
+
inputBatches: 0,
|
|
9452
|
+
outputBatches: 0,
|
|
9453
|
+
inputRows: 0,
|
|
9454
|
+
outputRows: 0,
|
|
9455
|
+
inputBytes: 0,
|
|
9456
|
+
outputBytes: 0
|
|
9457
|
+
};
|
|
9458
|
+
const token = dispatchHook?.onDispatchStart(info);
|
|
9459
|
+
let dispatchError;
|
|
9460
|
+
applyDefaults(params, method.defaults);
|
|
9461
|
+
try {
|
|
9462
|
+
if (method.type === "unary" /* UNARY */) {
|
|
9463
|
+
await dispatchUnary(method, params, writer, serverId, requestId, externalConfig, "tcp" /* TCP */);
|
|
9464
|
+
} else {
|
|
9465
|
+
await dispatchStream(method, params, writer, reader, serverId, requestId, externalConfig, "tcp" /* TCP */);
|
|
9466
|
+
}
|
|
9467
|
+
} catch (e) {
|
|
9468
|
+
dispatchError = e instanceof Error ? e : new Error(String(e));
|
|
9469
|
+
throw e;
|
|
9470
|
+
} finally {
|
|
9471
|
+
dispatchHook?.onDispatchEnd(token, info, stats, dispatchError);
|
|
9472
|
+
}
|
|
9473
|
+
}
|
|
9474
|
+
await new Promise((resolve, reject) => {
|
|
9475
|
+
server.listen({ host, port: requestedPort, backlog }, () => resolve());
|
|
9476
|
+
server.once("error", (err2) => reject(err2));
|
|
9477
|
+
});
|
|
9478
|
+
const address = server.address();
|
|
9479
|
+
const boundPort = typeof address === "object" && address ? address.port : requestedPort;
|
|
9480
|
+
options.onBound?.(host, boundPort);
|
|
9481
|
+
announcementSink.write(`TCP:${host}:${boundPort}
|
|
9482
|
+
`);
|
|
9483
|
+
if (idleTimeoutS > 0) {
|
|
9484
|
+
setTimeout(() => {
|
|
9485
|
+
if (activeConnections === 0 && !stopped)
|
|
9486
|
+
armIdleTimer();
|
|
9487
|
+
}, startupGraceS * 1000).unref?.();
|
|
9488
|
+
}
|
|
9489
|
+
return {
|
|
9490
|
+
host,
|
|
9491
|
+
port: boundPort,
|
|
9492
|
+
stop: shutdown,
|
|
9493
|
+
done
|
|
9494
|
+
};
|
|
9495
|
+
}
|
|
9239
9496
|
// src/launcher/serve-unix.ts
|
|
9240
9497
|
import { existsSync as existsSync2, unlinkSync as unlinkSync3 } from "node:fs";
|
|
9241
|
-
import { createServer } from "node:net";
|
|
9498
|
+
import { createServer as createServer2 } from "node:net";
|
|
9242
9499
|
import * as path2 from "node:path";
|
|
9243
|
-
var
|
|
9500
|
+
var EMPTY_SCHEMA7 = schema([]);
|
|
9244
9501
|
async function serveUnix(protocol, options) {
|
|
9245
9502
|
const sockPath = path2.resolve(options.unixPath);
|
|
9246
9503
|
const idleTimeoutS = options.idleTimeout ?? 300;
|
|
@@ -9277,7 +9534,7 @@ async function serveUnix(protocol, options) {
|
|
|
9277
9534
|
}
|
|
9278
9535
|
serveStartFired = true;
|
|
9279
9536
|
}
|
|
9280
|
-
const server =
|
|
9537
|
+
const server = createServer2({ allowHalfOpen: false });
|
|
9281
9538
|
let activeConnections = 0;
|
|
9282
9539
|
let idleTimer = null;
|
|
9283
9540
|
let resolveDone = () => {};
|
|
@@ -9366,8 +9623,8 @@ async function serveUnix(protocol, options) {
|
|
|
9366
9623
|
const { schema: schema2, batches } = stream;
|
|
9367
9624
|
if (batches.length === 0) {
|
|
9368
9625
|
const err2 = new RpcError("ProtocolError", "Request stream contains no batches", "");
|
|
9369
|
-
const errBatch = buildErrorBatch(
|
|
9370
|
-
await writer.writeStream(
|
|
9626
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, err2, serverId, null);
|
|
9627
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9371
9628
|
return;
|
|
9372
9629
|
}
|
|
9373
9630
|
const batch = batches[0];
|
|
@@ -9380,8 +9637,8 @@ async function serveUnix(protocol, options) {
|
|
|
9380
9637
|
params = parsed.params;
|
|
9381
9638
|
requestId = parsed.requestId;
|
|
9382
9639
|
} catch (e) {
|
|
9383
|
-
const errBatch = buildErrorBatch(
|
|
9384
|
-
await writer.writeStream(
|
|
9640
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, e, serverId, null);
|
|
9641
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9385
9642
|
if (e instanceof VersionError || e instanceof RpcError)
|
|
9386
9643
|
return;
|
|
9387
9644
|
throw e;
|
|
@@ -9396,8 +9653,8 @@ async function serveUnix(protocol, options) {
|
|
|
9396
9653
|
if (!method) {
|
|
9397
9654
|
const available = [...methods.keys()].sort();
|
|
9398
9655
|
const err2 = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
9399
|
-
const errBatch = buildErrorBatch(
|
|
9400
|
-
await writer.writeStream(
|
|
9656
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, err2, serverId, requestId);
|
|
9657
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9401
9658
|
return;
|
|
9402
9659
|
}
|
|
9403
9660
|
const methodType = method.type === "unary" /* UNARY */ ? "unary" : "stream";
|
|
@@ -9476,11 +9733,13 @@ export {
|
|
|
9476
9733
|
uint162 as uint16,
|
|
9477
9734
|
tryAcquireLock,
|
|
9478
9735
|
toSchema,
|
|
9736
|
+
tcpConnect,
|
|
9479
9737
|
subprocessConnect,
|
|
9480
9738
|
str,
|
|
9481
9739
|
statusRows,
|
|
9482
9740
|
socketPaths,
|
|
9483
9741
|
serveUnix,
|
|
9742
|
+
serveTcp,
|
|
9484
9743
|
resolveExternalLocation,
|
|
9485
9744
|
probeSocket,
|
|
9486
9745
|
pipeConnect,
|
|
@@ -9561,4 +9820,4 @@ export {
|
|
|
9561
9820
|
ARROW_CONTENT_TYPE
|
|
9562
9821
|
};
|
|
9563
9822
|
|
|
9564
|
-
//# debugId=
|
|
9823
|
+
//# debugId=EB2D2F8879345A8964756E2164756E21
|