@query-farm/vgi-rpc 0.7.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/client/index.d.ts +1 -1
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/tcp.d.ts +19 -0
- package/dist/client/tcp.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -1
- package/dist/index.core.d.ts +1 -0
- package/dist/index.core.d.ts.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1537 -1263
- package/dist/index.js.map +14 -11
- package/dist/launcher/index.d.ts +1 -0
- package/dist/launcher/index.d.ts.map +1 -1
- package/dist/launcher/serve-tcp.d.ts +66 -0
- package/dist/launcher/serve-tcp.d.ts.map +1 -0
- package/dist/serve-stream.d.ts +23 -0
- package/dist/serve-stream.d.ts.map +1 -0
- package/dist/server.d.ts +18 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/types.d.ts +6 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client/index.ts +1 -0
- package/src/client/tcp.ts +68 -0
- package/src/client/types.ts +8 -0
- package/src/index.core.ts +1 -0
- package/src/index.ts +6 -0
- package/src/launcher/index.ts +1 -0
- package/src/launcher/serve-tcp.ts +382 -0
- package/src/serve-stream.ts +38 -0
- package/src/server.ts +31 -10
- package/src/types.ts +5 -0
package/dist/index.js
CHANGED
|
@@ -734,130 +734,42 @@ var init_zstd = __esm(() => {
|
|
|
734
734
|
isBun = typeof globalThis.Bun !== "undefined";
|
|
735
735
|
});
|
|
736
736
|
|
|
737
|
-
// src/
|
|
738
|
-
|
|
739
|
-
function _loadWriteSync() {
|
|
740
|
-
const req = import.meta.require ?? globalThis.require ?? null;
|
|
741
|
-
if (!req) {
|
|
742
|
-
throw new Error("FdSink requires Node.js or Bun (node:fs.writeSync). For other runtimes, " + "supply a custom AccessLogSink that wraps console.log or your logger.");
|
|
743
|
-
}
|
|
744
|
-
return req(_NODE_FS_MOD).writeSync;
|
|
745
|
-
}
|
|
737
|
+
// src/client/tcp.ts
|
|
738
|
+
import { connect } from "node:net";
|
|
746
739
|
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
return Math.round(f * 100) / 100;
|
|
780
|
-
}
|
|
740
|
+
// src/client/pipe.ts
|
|
741
|
+
import {
|
|
742
|
+
Field,
|
|
743
|
+
makeData,
|
|
744
|
+
RecordBatch,
|
|
745
|
+
RecordBatchStreamWriter as RecordBatchStreamWriter2,
|
|
746
|
+
Schema,
|
|
747
|
+
Struct,
|
|
748
|
+
vectorFromArray
|
|
749
|
+
} from "@query-farm/apache-arrow";
|
|
750
|
+
|
|
751
|
+
// src/constants.ts
|
|
752
|
+
var RPC_METHOD_KEY = "vgi_rpc.method";
|
|
753
|
+
var LOG_LEVEL_KEY = "vgi_rpc.log_level";
|
|
754
|
+
var LOG_MESSAGE_KEY = "vgi_rpc.log_message";
|
|
755
|
+
var LOG_EXTRA_KEY = "vgi_rpc.log_extra";
|
|
756
|
+
var REQUEST_VERSION_KEY = "vgi_rpc.request_version";
|
|
757
|
+
var REQUEST_VERSION = "1";
|
|
758
|
+
var SERVER_ID_KEY = "vgi_rpc.server_id";
|
|
759
|
+
var REQUEST_ID_KEY = "vgi_rpc.request_id";
|
|
760
|
+
var PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
|
|
761
|
+
var DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
|
|
762
|
+
var PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
|
|
763
|
+
var DESCRIBE_VERSION = "4";
|
|
764
|
+
var PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
|
|
765
|
+
var DESCRIBE_METHOD_NAME = "__describe__";
|
|
766
|
+
var STATE_KEY = "vgi_rpc.stream_state#b64";
|
|
767
|
+
var CANCEL_KEY = "vgi_rpc.cancel";
|
|
768
|
+
var LOCATION_KEY = "vgi_rpc.location";
|
|
769
|
+
var LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
|
|
770
|
+
var RPC_ERROR_HEADER = "X-VGI-RPC-Error";
|
|
771
|
+
var ERROR_KIND_KEY = "vgi_rpc.error_kind";
|
|
781
772
|
|
|
782
|
-
class AccessLogHook {
|
|
783
|
-
sink;
|
|
784
|
-
serverVersion;
|
|
785
|
-
level;
|
|
786
|
-
constructor(sink, options = {}) {
|
|
787
|
-
this.sink = sink;
|
|
788
|
-
if (typeof options === "string") {
|
|
789
|
-
this.serverVersion = options;
|
|
790
|
-
this.level = "INFO";
|
|
791
|
-
} else {
|
|
792
|
-
this.serverVersion = options.serverVersion ?? "";
|
|
793
|
-
this.level = options.level ?? "INFO";
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
onDispatchStart(_info) {
|
|
797
|
-
const token = { startNs: process.hrtime.bigint() };
|
|
798
|
-
return token;
|
|
799
|
-
}
|
|
800
|
-
onDispatchEnd(token, info, stats, error) {
|
|
801
|
-
const t = token;
|
|
802
|
-
const durationMs = t ? roundTo2(Number(process.hrtime.bigint() - t.startNs) / 1e6) : 0;
|
|
803
|
-
const status = error ? "error" : "ok";
|
|
804
|
-
const errType = error ? error.type ?? error.constructor.name : "";
|
|
805
|
-
const errMsg = error?.message ?? "";
|
|
806
|
-
const protocol = info.protocol ?? "";
|
|
807
|
-
const rec = {
|
|
808
|
-
timestamp: rfc3339Utc(),
|
|
809
|
-
level: "INFO",
|
|
810
|
-
logger: "vgi_rpc.access",
|
|
811
|
-
message: `${protocol}.${info.method} ${status}`,
|
|
812
|
-
server_id: info.serverId,
|
|
813
|
-
protocol,
|
|
814
|
-
protocol_hash: info.protocolHash ?? "",
|
|
815
|
-
method: info.method,
|
|
816
|
-
method_type: info.methodType,
|
|
817
|
-
principal: info.principal ?? "",
|
|
818
|
-
auth_domain: info.authDomain ?? "",
|
|
819
|
-
authenticated: info.authenticated ?? false,
|
|
820
|
-
remote_addr: info.remoteAddr ?? "",
|
|
821
|
-
duration_ms: durationMs,
|
|
822
|
-
status,
|
|
823
|
-
error_type: errType
|
|
824
|
-
};
|
|
825
|
-
if (errMsg)
|
|
826
|
-
rec.error_message = errMsg;
|
|
827
|
-
if (this.serverVersion)
|
|
828
|
-
rec.server_version = this.serverVersion;
|
|
829
|
-
if (info.protocolVersion)
|
|
830
|
-
rec.protocol_version = info.protocolVersion;
|
|
831
|
-
if (info.requestId)
|
|
832
|
-
rec.request_id = info.requestId;
|
|
833
|
-
if (info.requestData && info.requestData.length > 0) {
|
|
834
|
-
const encoded = base64(info.requestData);
|
|
835
|
-
if (this.level === "DEBUG") {
|
|
836
|
-
rec.request_data = encoded;
|
|
837
|
-
} else {
|
|
838
|
-
rec.original_request_bytes = encoded.length;
|
|
839
|
-
rec.truncated = true;
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
if (info.methodType === "stream") {
|
|
843
|
-
rec.stream_id = info.streamId ?? "00000000000000000000000000000000";
|
|
844
|
-
}
|
|
845
|
-
if (info.cancelled)
|
|
846
|
-
rec.cancelled = true;
|
|
847
|
-
if (stats.inputBatches + stats.outputBatches + stats.inputRows + stats.outputRows + stats.inputBytes + stats.outputBytes !== 0) {
|
|
848
|
-
rec.input_batches = stats.inputBatches;
|
|
849
|
-
rec.output_batches = stats.outputBatches;
|
|
850
|
-
rec.input_rows = stats.inputRows;
|
|
851
|
-
rec.output_rows = stats.outputRows;
|
|
852
|
-
rec.input_bytes = stats.inputBytes;
|
|
853
|
-
rec.output_bytes = stats.outputBytes;
|
|
854
|
-
}
|
|
855
|
-
try {
|
|
856
|
-
this.sink.write(`${JSON.stringify(rec)}
|
|
857
|
-
`);
|
|
858
|
-
} catch {}
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
773
|
// src/errors.ts
|
|
862
774
|
class RpcError extends Error {
|
|
863
775
|
errorType;
|
|
@@ -927,49 +839,6 @@ class ServerDrainingError extends Error {
|
|
|
927
839
|
}
|
|
928
840
|
}
|
|
929
841
|
|
|
930
|
-
// src/auth.ts
|
|
931
|
-
class AuthContext {
|
|
932
|
-
domain;
|
|
933
|
-
authenticated;
|
|
934
|
-
principal;
|
|
935
|
-
claims;
|
|
936
|
-
constructor(domain, authenticated, principal, claims = {}) {
|
|
937
|
-
this.domain = domain;
|
|
938
|
-
this.authenticated = authenticated;
|
|
939
|
-
this.principal = principal;
|
|
940
|
-
this.claims = claims;
|
|
941
|
-
}
|
|
942
|
-
static anonymous() {
|
|
943
|
-
return new AuthContext("", false, null);
|
|
944
|
-
}
|
|
945
|
-
requireAuthenticated() {
|
|
946
|
-
if (!this.authenticated) {
|
|
947
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
948
|
-
}
|
|
949
|
-
}
|
|
950
|
-
}
|
|
951
|
-
// src/constants.ts
|
|
952
|
-
var RPC_METHOD_KEY = "vgi_rpc.method";
|
|
953
|
-
var LOG_LEVEL_KEY = "vgi_rpc.log_level";
|
|
954
|
-
var LOG_MESSAGE_KEY = "vgi_rpc.log_message";
|
|
955
|
-
var LOG_EXTRA_KEY = "vgi_rpc.log_extra";
|
|
956
|
-
var REQUEST_VERSION_KEY = "vgi_rpc.request_version";
|
|
957
|
-
var REQUEST_VERSION = "1";
|
|
958
|
-
var SERVER_ID_KEY = "vgi_rpc.server_id";
|
|
959
|
-
var REQUEST_ID_KEY = "vgi_rpc.request_id";
|
|
960
|
-
var PROTOCOL_NAME_KEY = "vgi_rpc.protocol_name";
|
|
961
|
-
var DESCRIBE_VERSION_KEY = "vgi_rpc.describe_version";
|
|
962
|
-
var PROTOCOL_HASH_KEY = "vgi_rpc.protocol_hash";
|
|
963
|
-
var DESCRIBE_VERSION = "4";
|
|
964
|
-
var PROTOCOL_VERSION_KEY = "vgi_rpc.protocol_version";
|
|
965
|
-
var DESCRIBE_METHOD_NAME = "__describe__";
|
|
966
|
-
var STATE_KEY = "vgi_rpc.stream_state#b64";
|
|
967
|
-
var CANCEL_KEY = "vgi_rpc.cancel";
|
|
968
|
-
var LOCATION_KEY = "vgi_rpc.location";
|
|
969
|
-
var LOCATION_SHA256_KEY = "vgi_rpc.location.sha256";
|
|
970
|
-
var RPC_ERROR_HEADER = "X-VGI-RPC-Error";
|
|
971
|
-
var ERROR_KIND_KEY = "vgi_rpc.error_kind";
|
|
972
|
-
|
|
973
842
|
// src/arrow/impl-arrowjs/index.ts
|
|
974
843
|
import {
|
|
975
844
|
Binary as A_Binary,
|
|
@@ -1509,63 +1378,6 @@ async function readRequestFromBody(body) {
|
|
|
1509
1378
|
return { schema: batch.schema, batch };
|
|
1510
1379
|
}
|
|
1511
1380
|
|
|
1512
|
-
// src/client/capabilities.ts
|
|
1513
|
-
var MAX_REQUEST_BYTES_HEADER = "VGI-Max-Request-Bytes";
|
|
1514
|
-
var UPLOAD_URL_HEADER = "VGI-Upload-URL-Support";
|
|
1515
|
-
var MAX_UPLOAD_BYTES_HEADER = "VGI-Max-Upload-Bytes";
|
|
1516
|
-
function parseHeaderInt(headers, name) {
|
|
1517
|
-
const raw = headers.get(name) ?? headers.get(name.toLowerCase());
|
|
1518
|
-
if (raw == null)
|
|
1519
|
-
return null;
|
|
1520
|
-
const parsed = Number.parseInt(raw, 10);
|
|
1521
|
-
return Number.isFinite(parsed) ? parsed : null;
|
|
1522
|
-
}
|
|
1523
|
-
function parseCapabilitiesFromHeaders(headers) {
|
|
1524
|
-
const uploadRaw = headers.get(UPLOAD_URL_HEADER) ?? headers.get(UPLOAD_URL_HEADER.toLowerCase());
|
|
1525
|
-
const uploadUrlSupport = uploadRaw === "true";
|
|
1526
|
-
let cacheExpiresAt = null;
|
|
1527
|
-
const cc = headers.get("Cache-Control") ?? headers.get("cache-control");
|
|
1528
|
-
if (cc) {
|
|
1529
|
-
for (const token of cc.split(",")) {
|
|
1530
|
-
const t = token.trim().toLowerCase();
|
|
1531
|
-
if (t.startsWith("max-age=")) {
|
|
1532
|
-
const seconds = Number.parseFloat(t.slice("max-age=".length));
|
|
1533
|
-
if (Number.isFinite(seconds)) {
|
|
1534
|
-
cacheExpiresAt = Date.now() + seconds * 1000;
|
|
1535
|
-
}
|
|
1536
|
-
break;
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
}
|
|
1540
|
-
return {
|
|
1541
|
-
maxRequestBytes: parseHeaderInt(headers, MAX_REQUEST_BYTES_HEADER),
|
|
1542
|
-
uploadUrlSupport,
|
|
1543
|
-
maxUploadBytes: parseHeaderInt(headers, MAX_UPLOAD_BYTES_HEADER),
|
|
1544
|
-
cacheExpiresAt
|
|
1545
|
-
};
|
|
1546
|
-
}
|
|
1547
|
-
function isCapabilitySnapshotFresh(snapshot) {
|
|
1548
|
-
if (!snapshot)
|
|
1549
|
-
return false;
|
|
1550
|
-
if (snapshot.cacheExpiresAt == null)
|
|
1551
|
-
return true;
|
|
1552
|
-
return Date.now() < snapshot.cacheExpiresAt;
|
|
1553
|
-
}
|
|
1554
|
-
|
|
1555
|
-
// src/client/introspect.ts
|
|
1556
|
-
import { Schema as ArrowSchema } from "@query-farm/apache-arrow";
|
|
1557
|
-
|
|
1558
|
-
// src/client/ipc.ts
|
|
1559
|
-
import {
|
|
1560
|
-
Binary,
|
|
1561
|
-
Bool,
|
|
1562
|
-
DataType,
|
|
1563
|
-
Float64,
|
|
1564
|
-
Int64,
|
|
1565
|
-
RecordBatchReader as RecordBatchReader3,
|
|
1566
|
-
Utf8
|
|
1567
|
-
} from "@query-farm/apache-arrow";
|
|
1568
|
-
|
|
1569
1381
|
// src/wire/reader.ts
|
|
1570
1382
|
import { RecordBatchReader as RecordBatchReader2 } from "@query-farm/apache-arrow";
|
|
1571
1383
|
|
|
@@ -1637,12 +1449,24 @@ class IpcStreamReader {
|
|
|
1637
1449
|
}
|
|
1638
1450
|
}
|
|
1639
1451
|
|
|
1452
|
+
// src/client/introspect.ts
|
|
1453
|
+
import { Schema as ArrowSchema } from "@query-farm/apache-arrow";
|
|
1454
|
+
|
|
1640
1455
|
// src/client/ipc.ts
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1456
|
+
import {
|
|
1457
|
+
Binary,
|
|
1458
|
+
Bool,
|
|
1459
|
+
DataType,
|
|
1460
|
+
Float64,
|
|
1461
|
+
Int64,
|
|
1462
|
+
RecordBatchReader as RecordBatchReader3,
|
|
1463
|
+
Utf8
|
|
1464
|
+
} from "@query-farm/apache-arrow";
|
|
1465
|
+
function inferArrowType(value) {
|
|
1466
|
+
if (typeof value === "string")
|
|
1467
|
+
return new Utf8;
|
|
1468
|
+
if (typeof value === "boolean")
|
|
1469
|
+
return new Bool;
|
|
1646
1470
|
if (typeof value === "bigint")
|
|
1647
1471
|
return new Int64;
|
|
1648
1472
|
if (typeof value === "number")
|
|
@@ -1853,1173 +1677,1380 @@ async function httpIntrospect(baseUrl, options) {
|
|
|
1853
1677
|
return parseDescribeResponse(batches);
|
|
1854
1678
|
}
|
|
1855
1679
|
|
|
1856
|
-
// src/client/
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1680
|
+
// src/client/pipe.ts
|
|
1681
|
+
class PipeIncrementalWriter {
|
|
1682
|
+
writer;
|
|
1683
|
+
writeFn;
|
|
1684
|
+
closed = false;
|
|
1685
|
+
constructor(writeFn, schema2) {
|
|
1686
|
+
this.writeFn = writeFn;
|
|
1687
|
+
this.writer = new RecordBatchStreamWriter2;
|
|
1688
|
+
this.writer.reset(undefined, schema2);
|
|
1689
|
+
this.drain();
|
|
1690
|
+
}
|
|
1691
|
+
write(batch) {
|
|
1692
|
+
if (this.closed)
|
|
1693
|
+
throw new Error("PipeIncrementalWriter already closed");
|
|
1694
|
+
this.writer._writeRecordBatch(batch);
|
|
1695
|
+
this.drain();
|
|
1696
|
+
}
|
|
1697
|
+
close() {
|
|
1698
|
+
if (this.closed)
|
|
1699
|
+
return;
|
|
1700
|
+
this.closed = true;
|
|
1701
|
+
const eos = new Uint8Array(new Int32Array([-1, 0]).buffer);
|
|
1702
|
+
this.writeFn(eos);
|
|
1703
|
+
}
|
|
1704
|
+
drain() {
|
|
1705
|
+
const values = this.writer._sink._values;
|
|
1706
|
+
for (const chunk of values) {
|
|
1707
|
+
this.writeFn(chunk);
|
|
1708
|
+
}
|
|
1709
|
+
values.length = 0;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
class PipeStreamSession {
|
|
1714
|
+
_reader;
|
|
1715
|
+
_writeFn;
|
|
1865
1716
|
_onLog;
|
|
1866
|
-
_pendingBatches;
|
|
1867
|
-
_finished;
|
|
1868
1717
|
_header;
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1718
|
+
_inputWriter = null;
|
|
1719
|
+
_inputSchema = null;
|
|
1720
|
+
_outputStreamOpened = false;
|
|
1721
|
+
_closed = false;
|
|
1722
|
+
_outputSchema;
|
|
1723
|
+
_releaseBusy;
|
|
1724
|
+
_setDrainPromise;
|
|
1873
1725
|
_externalConfig;
|
|
1874
|
-
_postFn;
|
|
1875
1726
|
constructor(opts) {
|
|
1876
|
-
this.
|
|
1877
|
-
this.
|
|
1878
|
-
this._method = opts.method;
|
|
1879
|
-
this._stateToken = opts.stateToken;
|
|
1880
|
-
this._outputSchema = opts.outputSchema;
|
|
1881
|
-
this._inputSchema = opts.inputSchema;
|
|
1727
|
+
this._reader = opts.reader;
|
|
1728
|
+
this._writeFn = opts.writeFn;
|
|
1882
1729
|
this._onLog = opts.onLog;
|
|
1883
|
-
this._pendingBatches = opts.pendingBatches;
|
|
1884
|
-
this._finished = opts.finished;
|
|
1885
1730
|
this._header = opts.header;
|
|
1886
|
-
this.
|
|
1887
|
-
this.
|
|
1888
|
-
this.
|
|
1889
|
-
this._authorization = opts.authorization;
|
|
1731
|
+
this._outputSchema = opts.outputSchema;
|
|
1732
|
+
this._releaseBusy = opts.releaseBusy;
|
|
1733
|
+
this._setDrainPromise = opts.setDrainPromise;
|
|
1890
1734
|
this._externalConfig = opts.externalConfig;
|
|
1891
|
-
this._postFn = opts.postFn;
|
|
1892
|
-
}
|
|
1893
|
-
async _post(url, body) {
|
|
1894
|
-
if (this._postFn)
|
|
1895
|
-
return this._postFn(url, body);
|
|
1896
|
-
return fetch(url, {
|
|
1897
|
-
method: "POST",
|
|
1898
|
-
headers: this._buildHeaders(),
|
|
1899
|
-
body: await this._prepareBody(body)
|
|
1900
|
-
});
|
|
1901
1735
|
}
|
|
1902
1736
|
get header() {
|
|
1903
1737
|
return this._header;
|
|
1904
1738
|
}
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
}
|
|
1920
|
-
async _prepareBody(content) {
|
|
1921
|
-
if (this._compressionLevel != null && this._compressFn) {
|
|
1922
|
-
return await this._compressFn(content, this._compressionLevel);
|
|
1739
|
+
async _readOutputBatch() {
|
|
1740
|
+
while (true) {
|
|
1741
|
+
const batch = await this._reader.readNextBatch();
|
|
1742
|
+
if (batch === null)
|
|
1743
|
+
return null;
|
|
1744
|
+
if (batch.numRows === 0) {
|
|
1745
|
+
if (isExternalLocationBatch(batch)) {
|
|
1746
|
+
return await resolveExternalLocation(batch, this._externalConfig);
|
|
1747
|
+
}
|
|
1748
|
+
if (dispatchLogOrError(batch, this._onLog)) {
|
|
1749
|
+
continue;
|
|
1750
|
+
}
|
|
1751
|
+
}
|
|
1752
|
+
return batch;
|
|
1923
1753
|
}
|
|
1924
|
-
return content;
|
|
1925
1754
|
}
|
|
1926
|
-
async
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1755
|
+
async _ensureOutputStream() {
|
|
1756
|
+
if (this._outputStreamOpened)
|
|
1757
|
+
return;
|
|
1758
|
+
this._outputStreamOpened = true;
|
|
1759
|
+
const schema2 = await this._reader.openNextStream();
|
|
1760
|
+
if (!schema2) {
|
|
1761
|
+
throw new RpcError("ProtocolError", "Expected output stream but got EOF", "");
|
|
1930
1762
|
}
|
|
1931
|
-
return body;
|
|
1932
1763
|
}
|
|
1933
1764
|
async exchange(input) {
|
|
1934
|
-
if (this.
|
|
1935
|
-
throw new RpcError("ProtocolError", "Stream
|
|
1765
|
+
if (this._closed) {
|
|
1766
|
+
throw new RpcError("ProtocolError", "Stream session is closed", "");
|
|
1936
1767
|
}
|
|
1768
|
+
let inputSchema;
|
|
1769
|
+
let batch;
|
|
1937
1770
|
if (input.length === 0) {
|
|
1938
|
-
|
|
1939
|
-
const
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
const
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1771
|
+
inputSchema = this._inputSchema ?? this._outputSchema;
|
|
1772
|
+
const children = inputSchema.fields.map((f) => {
|
|
1773
|
+
return makeData({ type: f.type, length: 0, nullCount: 0 });
|
|
1774
|
+
});
|
|
1775
|
+
const structType = new Struct(inputSchema.fields);
|
|
1776
|
+
const data = makeData({
|
|
1777
|
+
type: structType,
|
|
1778
|
+
length: 0,
|
|
1779
|
+
children,
|
|
1780
|
+
nullCount: 0
|
|
1781
|
+
});
|
|
1782
|
+
batch = new RecordBatch(inputSchema, data);
|
|
1783
|
+
} else {
|
|
1784
|
+
const keys = Object.keys(input[0]);
|
|
1785
|
+
const fields = keys.map((key) => {
|
|
1786
|
+
let sample;
|
|
1787
|
+
for (const row of input) {
|
|
1788
|
+
if (row[key] != null) {
|
|
1789
|
+
sample = row[key];
|
|
1790
|
+
break;
|
|
1791
|
+
}
|
|
1952
1792
|
}
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
return vectorFromArray(values, f.type).data[0];
|
|
1962
|
-
});
|
|
1963
|
-
const structType = new Struct(inputSchema.fields);
|
|
1964
|
-
const data = makeData({
|
|
1965
|
-
type: structType,
|
|
1966
|
-
length: input.length,
|
|
1967
|
-
children,
|
|
1968
|
-
nullCount: 0
|
|
1969
|
-
});
|
|
1970
|
-
const metadata = new Map;
|
|
1971
|
-
metadata.set(STATE_KEY, this._stateToken);
|
|
1972
|
-
const batch = new RecordBatch(inputSchema, data, metadata);
|
|
1973
|
-
return this._doExchange(inputSchema, [batch]);
|
|
1974
|
-
}
|
|
1975
|
-
async _doExchange(schema2, batches) {
|
|
1976
|
-
const body = serializeIpcStream(schema2, batches);
|
|
1977
|
-
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
1978
|
-
if (resp.status === 401) {
|
|
1979
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
1980
|
-
}
|
|
1981
|
-
const responseBody = await this._readResponse(resp);
|
|
1982
|
-
const { batches: responseBatches } = await readResponseBatches(responseBody);
|
|
1983
|
-
let resultRows = [];
|
|
1984
|
-
for (const batch of responseBatches) {
|
|
1985
|
-
if (batch.numRows === 0) {
|
|
1986
|
-
dispatchLogOrError(batch, this._onLog);
|
|
1987
|
-
const token2 = batch.metadata?.get(STATE_KEY);
|
|
1988
|
-
if (token2) {
|
|
1989
|
-
this._stateToken = token2;
|
|
1793
|
+
const arrowType = inferArrowType(sample);
|
|
1794
|
+
return new Field(key, arrowType, true);
|
|
1795
|
+
});
|
|
1796
|
+
inputSchema = new Schema(fields);
|
|
1797
|
+
if (this._inputSchema) {
|
|
1798
|
+
const cached = this._inputSchema;
|
|
1799
|
+
if (cached.fields.length !== inputSchema.fields.length || cached.fields.some((f, i) => f.name !== inputSchema.fields[i].name)) {
|
|
1800
|
+
throw new RpcError("ProtocolError", `Exchange input schema changed: expected [${cached.fields.map((f) => f.name).join(", ")}] ` + `but got [${inputSchema.fields.map((f) => f.name).join(", ")}]`, "");
|
|
1990
1801
|
}
|
|
1991
|
-
|
|
1802
|
+
} else {
|
|
1803
|
+
this._inputSchema = inputSchema;
|
|
1992
1804
|
}
|
|
1993
|
-
const
|
|
1994
|
-
|
|
1995
|
-
|
|
1805
|
+
const children = inputSchema.fields.map((f) => {
|
|
1806
|
+
const values = input.map((row) => row[f.name]);
|
|
1807
|
+
return vectorFromArray(values, f.type).data[0];
|
|
1808
|
+
});
|
|
1809
|
+
const structType = new Struct(inputSchema.fields);
|
|
1810
|
+
const data = makeData({
|
|
1811
|
+
type: structType,
|
|
1812
|
+
length: input.length,
|
|
1813
|
+
children,
|
|
1814
|
+
nullCount: 0
|
|
1815
|
+
});
|
|
1816
|
+
batch = new RecordBatch(inputSchema, data);
|
|
1817
|
+
}
|
|
1818
|
+
if (!this._inputWriter) {
|
|
1819
|
+
this._inputWriter = new PipeIncrementalWriter(this._writeFn, inputSchema);
|
|
1820
|
+
}
|
|
1821
|
+
this._inputWriter.write(batch);
|
|
1822
|
+
await this._ensureOutputStream();
|
|
1823
|
+
try {
|
|
1824
|
+
const outputBatch = await this._readOutputBatch();
|
|
1825
|
+
if (outputBatch === null) {
|
|
1826
|
+
return [];
|
|
1996
1827
|
}
|
|
1997
|
-
|
|
1828
|
+
return extractBatchRows(outputBatch);
|
|
1829
|
+
} catch (e) {
|
|
1830
|
+
await this._cleanup();
|
|
1831
|
+
throw e;
|
|
1998
1832
|
}
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
1833
|
+
}
|
|
1834
|
+
async _cleanup() {
|
|
1835
|
+
if (this._closed)
|
|
1836
|
+
return;
|
|
1837
|
+
this._closed = true;
|
|
1838
|
+
if (this._inputWriter) {
|
|
1839
|
+
this._inputWriter.close();
|
|
1840
|
+
this._inputWriter = null;
|
|
1841
|
+
}
|
|
1842
|
+
try {
|
|
1843
|
+
if (this._outputStreamOpened) {
|
|
1844
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
1845
|
+
}
|
|
1846
|
+
} catch {}
|
|
1847
|
+
this._releaseBusy();
|
|
2013
1848
|
}
|
|
2014
1849
|
async* [Symbol.asyncIterator]() {
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
1850
|
+
if (this._closed)
|
|
1851
|
+
return;
|
|
1852
|
+
try {
|
|
1853
|
+
const tickSchema = new Schema([]);
|
|
1854
|
+
this._inputWriter = new PipeIncrementalWriter(this._writeFn, tickSchema);
|
|
1855
|
+
const structType = new Struct(tickSchema.fields);
|
|
1856
|
+
const tickData = makeData({
|
|
1857
|
+
type: structType,
|
|
1858
|
+
length: 0,
|
|
1859
|
+
children: [],
|
|
1860
|
+
nullCount: 0
|
|
1861
|
+
});
|
|
1862
|
+
const tickBatch = new RecordBatch(tickSchema, tickData);
|
|
1863
|
+
while (true) {
|
|
1864
|
+
this._inputWriter.write(tickBatch);
|
|
1865
|
+
await this._ensureOutputStream();
|
|
1866
|
+
const outputBatch = await this._readOutputBatch();
|
|
1867
|
+
if (outputBatch === null) {
|
|
1868
|
+
break;
|
|
2022
1869
|
}
|
|
1870
|
+
yield extractBatchRows(outputBatch);
|
|
2023
1871
|
}
|
|
2024
|
-
|
|
1872
|
+
} finally {
|
|
1873
|
+
if (this._inputWriter) {
|
|
1874
|
+
this._inputWriter.close();
|
|
1875
|
+
this._inputWriter = null;
|
|
1876
|
+
}
|
|
1877
|
+
try {
|
|
1878
|
+
if (this._outputStreamOpened) {
|
|
1879
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
1880
|
+
}
|
|
1881
|
+
} catch {}
|
|
1882
|
+
this._closed = true;
|
|
1883
|
+
this._releaseBusy();
|
|
2025
1884
|
}
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
if (this._stateToken === null)
|
|
1885
|
+
}
|
|
1886
|
+
close() {
|
|
1887
|
+
if (this._closed)
|
|
2030
1888
|
return;
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
const
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
if (isExternalLocationBatch(batch)) {
|
|
2047
|
-
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2048
|
-
} else {
|
|
2049
|
-
dispatchLogOrError(batch, this._onLog);
|
|
2050
|
-
continue;
|
|
1889
|
+
this._closed = true;
|
|
1890
|
+
if (this._inputWriter) {
|
|
1891
|
+
this._inputWriter.close();
|
|
1892
|
+
this._inputWriter = null;
|
|
1893
|
+
} else {
|
|
1894
|
+
const emptySchema = new Schema([]);
|
|
1895
|
+
const ipc = serializeIpcStream(emptySchema, []);
|
|
1896
|
+
this._writeFn(ipc);
|
|
1897
|
+
}
|
|
1898
|
+
const drainPromise = (async () => {
|
|
1899
|
+
try {
|
|
1900
|
+
if (!this._outputStreamOpened) {
|
|
1901
|
+
const schema2 = await this._reader.openNextStream();
|
|
1902
|
+
if (schema2) {
|
|
1903
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
2051
1904
|
}
|
|
1905
|
+
} else {
|
|
1906
|
+
while (await this._reader.readNextBatch() !== null) {}
|
|
2052
1907
|
}
|
|
2053
|
-
|
|
1908
|
+
} catch {} finally {
|
|
1909
|
+
this._releaseBusy();
|
|
2054
1910
|
}
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
}
|
|
1911
|
+
})();
|
|
1912
|
+
this._setDrainPromise(drainPromise);
|
|
2058
1913
|
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
1914
|
+
}
|
|
1915
|
+
function pipeConnect(readable, writable, options) {
|
|
1916
|
+
const onLog = options?.onLog;
|
|
1917
|
+
const externalConfig = options?.externalLocation;
|
|
1918
|
+
let reader = null;
|
|
1919
|
+
let readerPromise = null;
|
|
1920
|
+
let methodCache = null;
|
|
1921
|
+
let protocolName = "";
|
|
1922
|
+
let serverProtocolVersion = "";
|
|
1923
|
+
let _busy = false;
|
|
1924
|
+
let _drainPromise = null;
|
|
1925
|
+
let closed = false;
|
|
1926
|
+
const writeFn = (bytes) => {
|
|
1927
|
+
writable.write(bytes);
|
|
1928
|
+
writable.flush?.();
|
|
1929
|
+
};
|
|
1930
|
+
async function ensureReader() {
|
|
1931
|
+
if (reader)
|
|
1932
|
+
return reader;
|
|
1933
|
+
if (!readerPromise) {
|
|
1934
|
+
readerPromise = IpcStreamReader.create(readable);
|
|
2075
1935
|
}
|
|
2076
|
-
|
|
1936
|
+
reader = await readerPromise;
|
|
1937
|
+
return reader;
|
|
2077
1938
|
}
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
const headers = { "Content-Type": ARROW_CONTENT_TYPE };
|
|
2088
|
-
if (authorization)
|
|
2089
|
-
headers.Authorization = authorization;
|
|
2090
|
-
const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
|
|
2091
|
-
method: "POST",
|
|
2092
|
-
headers,
|
|
2093
|
-
body
|
|
2094
|
-
});
|
|
2095
|
-
if (resp.status === 404) {
|
|
2096
|
-
throw new RpcError("NotSupported", "Server does not support upload URLs", "");
|
|
1939
|
+
async function acquireBusy() {
|
|
1940
|
+
if (_drainPromise) {
|
|
1941
|
+
await _drainPromise;
|
|
1942
|
+
_drainPromise = null;
|
|
1943
|
+
}
|
|
1944
|
+
if (_busy) {
|
|
1945
|
+
throw new Error("Pipe transport is busy — another call or stream is in progress. " + "Pipe connections are single-threaded; wait for the current operation to complete.");
|
|
1946
|
+
}
|
|
1947
|
+
_busy = true;
|
|
2097
1948
|
}
|
|
2098
|
-
|
|
2099
|
-
|
|
1949
|
+
function releaseBusy() {
|
|
1950
|
+
_busy = false;
|
|
2100
1951
|
}
|
|
2101
|
-
|
|
2102
|
-
|
|
1952
|
+
function setDrainPromise(p) {
|
|
1953
|
+
_drainPromise = p;
|
|
2103
1954
|
}
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
const
|
|
2113
|
-
const
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
if (expiresRaw instanceof Date) {
|
|
2117
|
-
expiresAt = expiresRaw;
|
|
2118
|
-
} else if (typeof expiresRaw === "bigint") {
|
|
2119
|
-
expiresAt = new Date(Number(expiresRaw / 1000n));
|
|
2120
|
-
} else if (typeof expiresRaw === "number") {
|
|
2121
|
-
expiresAt = new Date(expiresRaw);
|
|
2122
|
-
} else {
|
|
2123
|
-
expiresAt = new Date;
|
|
1955
|
+
async function ensureMethodCache() {
|
|
1956
|
+
if (methodCache)
|
|
1957
|
+
return methodCache;
|
|
1958
|
+
await acquireBusy();
|
|
1959
|
+
try {
|
|
1960
|
+
const emptySchema = new Schema([]);
|
|
1961
|
+
const body = buildRequestIpc(emptySchema, {}, DESCRIBE_METHOD_NAME);
|
|
1962
|
+
writeFn(body);
|
|
1963
|
+
const r = await ensureReader();
|
|
1964
|
+
const response = await r.readStream();
|
|
1965
|
+
if (!response) {
|
|
1966
|
+
throw new Error("EOF reading __describe__ response");
|
|
2124
1967
|
}
|
|
2125
|
-
|
|
1968
|
+
const desc = await parseDescribeResponse(response.batches, onLog);
|
|
1969
|
+
protocolName = desc.protocolName;
|
|
1970
|
+
serverProtocolVersion = desc.protocolVersion;
|
|
1971
|
+
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
1972
|
+
return methodCache;
|
|
1973
|
+
} finally {
|
|
1974
|
+
releaseBusy();
|
|
2126
1975
|
}
|
|
2127
1976
|
}
|
|
2128
|
-
if (pairs.length === 0) {
|
|
2129
|
-
throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
|
|
2130
|
-
}
|
|
2131
|
-
return pairs;
|
|
2132
|
-
}
|
|
2133
|
-
async function buildPointerRequestBody(originalBody, downloadUrl) {
|
|
2134
|
-
const reader = await RecordBatchReader4.from(originalBody);
|
|
2135
|
-
await reader.open();
|
|
2136
|
-
const schema2 = reader.schema;
|
|
2137
|
-
if (!schema2) {
|
|
2138
|
-
throw new RpcError("ProtocolError", "Original request body has no schema", "");
|
|
2139
|
-
}
|
|
2140
|
-
const batches = reader.readAll();
|
|
2141
|
-
if (batches.length === 0) {
|
|
2142
|
-
throw new RpcError("ProtocolError", "Original request body has no batches", "");
|
|
2143
|
-
}
|
|
2144
|
-
const original = batches[0];
|
|
2145
|
-
const originalMeta = original.metadata ?? new Map;
|
|
2146
|
-
const pointer = makeExternalLocationBatch(schema2, downloadUrl);
|
|
2147
|
-
const merged = new Map(pointer.metadata ?? new Map);
|
|
2148
|
-
const method = originalMeta.get(RPC_METHOD_KEY);
|
|
2149
|
-
const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
|
|
2150
|
-
if (method)
|
|
2151
|
-
merged.set(RPC_METHOD_KEY, method);
|
|
2152
|
-
merged.set(REQUEST_VERSION_KEY, version);
|
|
2153
|
-
for (const [k, v] of originalMeta) {
|
|
2154
|
-
if (!merged.has(k))
|
|
2155
|
-
merged.set(k, v);
|
|
2156
|
-
}
|
|
2157
|
-
const { RecordBatch: RecordBatch2 } = await import("@query-farm/apache-arrow");
|
|
2158
|
-
const pointerWithMeta = new RecordBatch2(schema2, pointer.data, merged);
|
|
2159
|
-
return serializeIpcStream(schema2, [pointerWithMeta]);
|
|
2160
|
-
}
|
|
2161
|
-
async function externalizeRequestBody(body, opts) {
|
|
2162
|
-
const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
|
|
2163
|
-
const pair = pairs[0];
|
|
2164
|
-
if (opts.urlValidator) {
|
|
2165
|
-
opts.urlValidator(pair.uploadUrl);
|
|
2166
|
-
opts.urlValidator(pair.downloadUrl);
|
|
2167
|
-
}
|
|
2168
|
-
const putResp = await fetch(pair.uploadUrl, {
|
|
2169
|
-
method: "PUT",
|
|
2170
|
-
headers: { "Content-Type": ARROW_CONTENT_TYPE },
|
|
2171
|
-
body
|
|
2172
|
-
});
|
|
2173
|
-
if (!putResp.ok) {
|
|
2174
|
-
throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
|
|
2175
|
-
}
|
|
2176
|
-
return buildPointerRequestBody(body, pair.downloadUrl);
|
|
2177
|
-
}
|
|
2178
|
-
|
|
2179
|
-
// src/client/connect.ts
|
|
2180
|
-
function httpConnect(baseUrl, options) {
|
|
2181
|
-
const prefix = (options?.prefix ?? "").replace(/\/+$/, "");
|
|
2182
|
-
const onLog = options?.onLog;
|
|
2183
|
-
const compressionLevel = options?.compressionLevel;
|
|
2184
|
-
const authorization = options?.authorization;
|
|
2185
|
-
const externalConfig = options?.externalLocation;
|
|
2186
|
-
let methodCache = null;
|
|
2187
|
-
let serverProtocolVersion = "";
|
|
2188
|
-
let compressFn;
|
|
2189
|
-
let decompressFn;
|
|
2190
|
-
let compressionLoaded = false;
|
|
2191
|
-
let capabilities = null;
|
|
2192
|
-
function updateCapabilitiesFromResponse(resp) {
|
|
2193
|
-
const next = parseCapabilitiesFromHeaders(resp.headers);
|
|
2194
|
-
if (next.maxRequestBytes != null || next.uploadUrlSupport) {
|
|
2195
|
-
capabilities = next;
|
|
2196
|
-
}
|
|
2197
|
-
}
|
|
2198
|
-
async function maybeExternalize(body) {
|
|
2199
|
-
const caps = isCapabilitySnapshotFresh(capabilities) ? capabilities : null;
|
|
2200
|
-
if (!caps)
|
|
2201
|
-
return body;
|
|
2202
|
-
if (!caps.uploadUrlSupport)
|
|
2203
|
-
return body;
|
|
2204
|
-
if (caps.maxRequestBytes == null || body.byteLength <= caps.maxRequestBytes)
|
|
2205
|
-
return body;
|
|
2206
|
-
return externalizeRequestBody(body, {
|
|
2207
|
-
baseUrl,
|
|
2208
|
-
prefix,
|
|
2209
|
-
authorization,
|
|
2210
|
-
urlValidator: externalConfig?.urlValidator ?? null
|
|
2211
|
-
});
|
|
2212
|
-
}
|
|
2213
|
-
async function postWithExternalization(url, body) {
|
|
2214
|
-
const sendBody = await maybeExternalize(body);
|
|
2215
|
-
let resp = await fetch(url, {
|
|
2216
|
-
method: "POST",
|
|
2217
|
-
headers: buildHeaders(),
|
|
2218
|
-
body: await prepareBody(sendBody)
|
|
2219
|
-
});
|
|
2220
|
-
updateCapabilitiesFromResponse(resp);
|
|
2221
|
-
if (resp.status === 413 && capabilities?.uploadUrlSupport && body.byteLength > 0) {
|
|
2222
|
-
const externalized = await externalizeRequestBody(body, {
|
|
2223
|
-
baseUrl,
|
|
2224
|
-
prefix,
|
|
2225
|
-
authorization,
|
|
2226
|
-
urlValidator: externalConfig?.urlValidator ?? null
|
|
2227
|
-
});
|
|
2228
|
-
resp = await fetch(url, {
|
|
2229
|
-
method: "POST",
|
|
2230
|
-
headers: buildHeaders(),
|
|
2231
|
-
body: await prepareBody(externalized)
|
|
2232
|
-
});
|
|
2233
|
-
updateCapabilitiesFromResponse(resp);
|
|
2234
|
-
}
|
|
2235
|
-
return resp;
|
|
2236
|
-
}
|
|
2237
|
-
async function ensureCompression() {
|
|
2238
|
-
if (compressionLoaded || compressionLevel == null)
|
|
2239
|
-
return;
|
|
2240
|
-
try {
|
|
2241
|
-
const mod = await Promise.resolve().then(() => (init_zstd(), exports_zstd));
|
|
2242
|
-
compressFn = mod.zstdCompress;
|
|
2243
|
-
decompressFn = mod.zstdDecompress;
|
|
2244
|
-
} catch {}
|
|
2245
|
-
compressionLoaded = true;
|
|
2246
|
-
}
|
|
2247
|
-
function buildHeaders() {
|
|
2248
|
-
const headers = {
|
|
2249
|
-
"Content-Type": ARROW_CONTENT_TYPE
|
|
2250
|
-
};
|
|
2251
|
-
if (compressionLevel != null && compressFn) {
|
|
2252
|
-
headers["Content-Encoding"] = "zstd";
|
|
2253
|
-
}
|
|
2254
|
-
if (compressionLevel != null && decompressFn) {
|
|
2255
|
-
headers["Accept-Encoding"] = "zstd";
|
|
2256
|
-
}
|
|
2257
|
-
if (authorization) {
|
|
2258
|
-
headers.Authorization = authorization;
|
|
2259
|
-
}
|
|
2260
|
-
return headers;
|
|
2261
|
-
}
|
|
2262
|
-
async function prepareBody(content) {
|
|
2263
|
-
if (compressionLevel != null && compressFn) {
|
|
2264
|
-
return await compressFn(content, compressionLevel);
|
|
2265
|
-
}
|
|
2266
|
-
return content;
|
|
2267
|
-
}
|
|
2268
|
-
function checkAuth(resp) {
|
|
2269
|
-
if (resp.status === 401) {
|
|
2270
|
-
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2271
|
-
}
|
|
2272
|
-
}
|
|
2273
|
-
async function readResponse(resp) {
|
|
2274
|
-
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2275
|
-
if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
|
|
2276
|
-
body = new Uint8Array(await decompressFn(body));
|
|
2277
|
-
}
|
|
2278
|
-
return body;
|
|
2279
|
-
}
|
|
2280
|
-
async function ensureMethodCache() {
|
|
2281
|
-
if (methodCache)
|
|
2282
|
-
return methodCache;
|
|
2283
|
-
await ensureCompression();
|
|
2284
|
-
const desc = await httpIntrospect(baseUrl, {
|
|
2285
|
-
prefix,
|
|
2286
|
-
authorization,
|
|
2287
|
-
compressionLevel,
|
|
2288
|
-
compressFn,
|
|
2289
|
-
decompressFn
|
|
2290
|
-
});
|
|
2291
|
-
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2292
|
-
serverProtocolVersion = desc.protocolVersion;
|
|
2293
|
-
return methodCache;
|
|
2294
|
-
}
|
|
2295
1977
|
return {
|
|
2296
1978
|
async call(method, params) {
|
|
2297
|
-
await ensureCompression();
|
|
2298
1979
|
const methods = await ensureMethodCache();
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2305
|
-
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}`, body);
|
|
2306
|
-
checkAuth(resp);
|
|
2307
|
-
const responseBody = await readResponse(resp);
|
|
2308
|
-
const { batches } = await readResponseBatches(responseBody);
|
|
2309
|
-
let resultBatch = null;
|
|
2310
|
-
for (let batch of batches) {
|
|
2311
|
-
if (batch.numRows === 0) {
|
|
2312
|
-
if (isExternalLocationBatch(batch)) {
|
|
2313
|
-
batch = await resolveExternalLocation(batch, externalConfig);
|
|
2314
|
-
} else {
|
|
2315
|
-
dispatchLogOrError(batch, onLog);
|
|
2316
|
-
continue;
|
|
2317
|
-
}
|
|
1980
|
+
await acquireBusy();
|
|
1981
|
+
try {
|
|
1982
|
+
const info = methods.get(method);
|
|
1983
|
+
if (!info) {
|
|
1984
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2318
1985
|
}
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
await ensureCompression();
|
|
2334
|
-
const methods = await ensureMethodCache();
|
|
2335
|
-
const info = methods.get(method);
|
|
2336
|
-
if (!info) {
|
|
2337
|
-
throw new Error(`Unknown method: '${method}'`);
|
|
2338
|
-
}
|
|
2339
|
-
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2340
|
-
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2341
|
-
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}/init`, body);
|
|
2342
|
-
checkAuth(resp);
|
|
2343
|
-
const responseBody = await readResponse(resp);
|
|
2344
|
-
let header = null;
|
|
2345
|
-
let stateToken = null;
|
|
2346
|
-
const pendingBatches = [];
|
|
2347
|
-
let finished = false;
|
|
2348
|
-
let streamSchema = null;
|
|
2349
|
-
if (info.headerSchema) {
|
|
2350
|
-
const reader = await readSequentialStreams(responseBody);
|
|
2351
|
-
const headerStream = await reader.readStream();
|
|
2352
|
-
if (headerStream) {
|
|
2353
|
-
for (const batch of headerStream.batches) {
|
|
2354
|
-
if (batch.numRows === 0) {
|
|
1986
|
+
const r = await ensureReader();
|
|
1987
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
1988
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
1989
|
+
writeFn(body);
|
|
1990
|
+
const response = await r.readStream();
|
|
1991
|
+
if (!response) {
|
|
1992
|
+
throw new Error("EOF reading response");
|
|
1993
|
+
}
|
|
1994
|
+
let resultBatch = null;
|
|
1995
|
+
for (let batch of response.batches) {
|
|
1996
|
+
if (batch.numRows === 0) {
|
|
1997
|
+
if (isExternalLocationBatch(batch)) {
|
|
1998
|
+
batch = await resolveExternalLocation(batch, externalConfig);
|
|
1999
|
+
} else {
|
|
2355
2000
|
dispatchLogOrError(batch, onLog);
|
|
2356
2001
|
continue;
|
|
2357
2002
|
}
|
|
2358
|
-
const rows = extractBatchRows(batch);
|
|
2359
|
-
if (rows.length > 0) {
|
|
2360
|
-
header = rows[0];
|
|
2361
|
-
}
|
|
2362
2003
|
}
|
|
2004
|
+
resultBatch = batch;
|
|
2363
2005
|
}
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
streamSchema = dataStream.schema;
|
|
2006
|
+
if (!resultBatch) {
|
|
2007
|
+
return null;
|
|
2367
2008
|
}
|
|
2368
|
-
const
|
|
2369
|
-
if (
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2009
|
+
const rows = extractBatchRows(resultBatch);
|
|
2010
|
+
if (rows.length === 0)
|
|
2011
|
+
return null;
|
|
2012
|
+
if (info.resultSchema.fields.length === 0)
|
|
2013
|
+
return null;
|
|
2014
|
+
return rows[0];
|
|
2015
|
+
} finally {
|
|
2016
|
+
releaseBusy();
|
|
2017
|
+
}
|
|
2018
|
+
},
|
|
2019
|
+
async stream(method, params) {
|
|
2020
|
+
const methods = await ensureMethodCache();
|
|
2021
|
+
await acquireBusy();
|
|
2022
|
+
try {
|
|
2023
|
+
const info = methods.get(method);
|
|
2024
|
+
if (!info) {
|
|
2025
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2026
|
+
}
|
|
2027
|
+
const r = await ensureReader();
|
|
2028
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2029
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2030
|
+
writeFn(body);
|
|
2031
|
+
let header = null;
|
|
2032
|
+
if (info.headerSchema) {
|
|
2033
|
+
const headerStream = await r.readStream();
|
|
2034
|
+
if (headerStream) {
|
|
2035
|
+
for (const batch of headerStream.batches) {
|
|
2036
|
+
if (batch.numRows === 0) {
|
|
2037
|
+
dispatchLogOrError(batch, onLog);
|
|
2375
2038
|
continue;
|
|
2376
2039
|
}
|
|
2377
|
-
const
|
|
2378
|
-
if (
|
|
2379
|
-
|
|
2380
|
-
continue;
|
|
2040
|
+
const rows = extractBatchRows(batch);
|
|
2041
|
+
if (rows.length > 0) {
|
|
2042
|
+
header = rows[0];
|
|
2381
2043
|
}
|
|
2382
|
-
dispatchLogOrError(batch, onLog);
|
|
2383
|
-
continue;
|
|
2384
2044
|
}
|
|
2385
|
-
pendingBatches.push(batch);
|
|
2386
2045
|
}
|
|
2387
2046
|
}
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
continue;
|
|
2410
|
-
}
|
|
2411
|
-
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2412
|
-
if (level === "EXCEPTION") {
|
|
2413
|
-
errorBatches.push(batch);
|
|
2414
|
-
continue;
|
|
2415
|
-
}
|
|
2416
|
-
dispatchLogOrError(batch, onLog);
|
|
2417
|
-
continue;
|
|
2418
|
-
}
|
|
2419
|
-
pendingBatches.push(batch);
|
|
2420
|
-
}
|
|
2421
|
-
if (errorBatches.length > 0) {
|
|
2422
|
-
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2423
|
-
pendingBatches.push(...errorBatches);
|
|
2424
|
-
} else {
|
|
2425
|
-
for (const batch of errorBatches) {
|
|
2426
|
-
dispatchLogOrError(batch, onLog);
|
|
2427
|
-
}
|
|
2428
|
-
}
|
|
2429
|
-
}
|
|
2430
|
-
}
|
|
2431
|
-
if (pendingBatches.length === 0 && stateToken === null) {
|
|
2432
|
-
finished = true;
|
|
2047
|
+
const outputSchema = info.outputSchema ?? info.resultSchema;
|
|
2048
|
+
return new PipeStreamSession({
|
|
2049
|
+
reader: r,
|
|
2050
|
+
writeFn,
|
|
2051
|
+
onLog,
|
|
2052
|
+
header,
|
|
2053
|
+
outputSchema,
|
|
2054
|
+
releaseBusy,
|
|
2055
|
+
setDrainPromise,
|
|
2056
|
+
externalConfig
|
|
2057
|
+
});
|
|
2058
|
+
} catch (e) {
|
|
2059
|
+
try {
|
|
2060
|
+
const r = await ensureReader();
|
|
2061
|
+
const emptySchema = new Schema([]);
|
|
2062
|
+
const ipc = serializeIpcStream(emptySchema, []);
|
|
2063
|
+
writeFn(ipc);
|
|
2064
|
+
const outStream = await r.readStream();
|
|
2065
|
+
} catch {}
|
|
2066
|
+
releaseBusy();
|
|
2067
|
+
throw e;
|
|
2433
2068
|
}
|
|
2434
|
-
const outputSchema = (streamSchema && streamSchema.fields.length > 0 ? streamSchema : null) ?? (pendingBatches.length > 0 ? pendingBatches[0].schema : null) ?? info.outputSchema ?? info.resultSchema;
|
|
2435
|
-
return new HttpStreamSession({
|
|
2436
|
-
baseUrl,
|
|
2437
|
-
prefix,
|
|
2438
|
-
method,
|
|
2439
|
-
stateToken,
|
|
2440
|
-
outputSchema,
|
|
2441
|
-
inputSchema: info.inputSchema,
|
|
2442
|
-
onLog,
|
|
2443
|
-
pendingBatches,
|
|
2444
|
-
finished,
|
|
2445
|
-
header,
|
|
2446
|
-
compressionLevel,
|
|
2447
|
-
compressFn,
|
|
2448
|
-
decompressFn,
|
|
2449
|
-
authorization,
|
|
2450
|
-
externalConfig,
|
|
2451
|
-
postFn: postWithExternalization
|
|
2452
|
-
});
|
|
2453
2069
|
},
|
|
2454
2070
|
async describe() {
|
|
2455
|
-
await
|
|
2456
|
-
return
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
decompressFn
|
|
2462
|
-
});
|
|
2071
|
+
const methods = await ensureMethodCache();
|
|
2072
|
+
return {
|
|
2073
|
+
protocolName,
|
|
2074
|
+
protocolVersion: serverProtocolVersion,
|
|
2075
|
+
methods: [...methods.values()]
|
|
2076
|
+
};
|
|
2463
2077
|
},
|
|
2464
|
-
close() {
|
|
2078
|
+
close() {
|
|
2079
|
+
if (closed)
|
|
2080
|
+
return;
|
|
2081
|
+
closed = true;
|
|
2082
|
+
writable.end();
|
|
2083
|
+
}
|
|
2465
2084
|
};
|
|
2466
2085
|
}
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2086
|
+
function subprocessConnect(cmd, options) {
|
|
2087
|
+
const proc = Bun.spawn(cmd, {
|
|
2088
|
+
stdin: "pipe",
|
|
2089
|
+
stdout: "pipe",
|
|
2090
|
+
stderr: options?.stderr ?? "ignore",
|
|
2091
|
+
cwd: options?.cwd,
|
|
2092
|
+
env: options?.env ? { ...process.env, ...options.env } : undefined
|
|
2093
|
+
});
|
|
2094
|
+
const stdout = proc.stdout;
|
|
2095
|
+
const writable = {
|
|
2096
|
+
write(data) {
|
|
2097
|
+
proc.stdin.write(data);
|
|
2098
|
+
},
|
|
2099
|
+
flush() {
|
|
2100
|
+
proc.stdin.flush();
|
|
2101
|
+
},
|
|
2102
|
+
end() {
|
|
2103
|
+
proc.stdin.end();
|
|
2104
|
+
}
|
|
2472
2105
|
};
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
if (json.resource_tos_uri)
|
|
2486
|
-
result.resourceTosUri = json.resource_tos_uri;
|
|
2487
|
-
if (json.client_id)
|
|
2488
|
-
result.clientId = json.client_id;
|
|
2489
|
-
if (json.client_secret)
|
|
2490
|
-
result.clientSecret = json.client_secret;
|
|
2491
|
-
if (json.use_id_token_as_bearer)
|
|
2492
|
-
result.useIdTokenAsBearer = json.use_id_token_as_bearer;
|
|
2493
|
-
if (json.device_code_client_id)
|
|
2494
|
-
result.deviceCodeClientId = json.device_code_client_id;
|
|
2495
|
-
if (json.device_code_client_secret)
|
|
2496
|
-
result.deviceCodeClientSecret = json.device_code_client_secret;
|
|
2497
|
-
return result;
|
|
2106
|
+
const client = pipeConnect(stdout, writable, {
|
|
2107
|
+
onLog: options?.onLog,
|
|
2108
|
+
externalLocation: options?.externalLocation
|
|
2109
|
+
});
|
|
2110
|
+
const originalClose = client.close;
|
|
2111
|
+
client.close = () => {
|
|
2112
|
+
originalClose.call(client);
|
|
2113
|
+
try {
|
|
2114
|
+
proc.kill();
|
|
2115
|
+
} catch {}
|
|
2116
|
+
};
|
|
2117
|
+
return client;
|
|
2498
2118
|
}
|
|
2499
|
-
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2119
|
+
|
|
2120
|
+
// src/client/tcp.ts
|
|
2121
|
+
function tcpConnect(host, port, options) {
|
|
2122
|
+
const socket = connect({ host, port });
|
|
2123
|
+
socket.setNoDelay(true);
|
|
2124
|
+
socket.on("error", () => {});
|
|
2125
|
+
const readable = socket;
|
|
2126
|
+
const writable = {
|
|
2127
|
+
write(data) {
|
|
2128
|
+
socket.write(data);
|
|
2129
|
+
},
|
|
2130
|
+
end() {
|
|
2131
|
+
socket.end();
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
const client = pipeConnect(readable, writable, {
|
|
2135
|
+
onLog: options?.onLog,
|
|
2136
|
+
externalLocation: options?.externalLocation
|
|
2137
|
+
});
|
|
2138
|
+
const originalClose = client.close;
|
|
2139
|
+
client.close = () => {
|
|
2140
|
+
originalClose.call(client);
|
|
2141
|
+
try {
|
|
2142
|
+
socket.destroy();
|
|
2143
|
+
} catch {}
|
|
2144
|
+
};
|
|
2145
|
+
return client;
|
|
2507
2146
|
}
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2147
|
+
// src/access-log.ts
|
|
2148
|
+
var _NODE_FS_MOD = "node:fs";
|
|
2149
|
+
function _loadWriteSync() {
|
|
2150
|
+
const req = import.meta.require ?? globalThis.require ?? null;
|
|
2151
|
+
if (!req) {
|
|
2152
|
+
throw new Error("FdSink requires Node.js or Bun (node:fs.writeSync). For other runtimes, " + "supply a custom AccessLogSink that wraps console.log or your logger.");
|
|
2512
2153
|
}
|
|
2513
|
-
|
|
2514
|
-
return parseMetadataJson(json);
|
|
2515
|
-
}
|
|
2516
|
-
function parseResourceMetadataUrl(wwwAuthenticate) {
|
|
2517
|
-
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
2518
|
-
if (!bearerMatch)
|
|
2519
|
-
return null;
|
|
2520
|
-
const params = bearerMatch[1];
|
|
2521
|
-
const metadataMatch = params.match(/resource_metadata="([^"]+)"/);
|
|
2522
|
-
if (!metadataMatch)
|
|
2523
|
-
return null;
|
|
2524
|
-
return metadataMatch[1];
|
|
2154
|
+
return req(_NODE_FS_MOD).writeSync;
|
|
2525
2155
|
}
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2156
|
+
|
|
2157
|
+
class FdSink {
|
|
2158
|
+
fd;
|
|
2159
|
+
_writeSync = _loadWriteSync();
|
|
2160
|
+
constructor(fd) {
|
|
2161
|
+
this.fd = fd;
|
|
2162
|
+
}
|
|
2163
|
+
write(line) {
|
|
2164
|
+
const buf = new TextEncoder().encode(line);
|
|
2165
|
+
let offset = 0;
|
|
2166
|
+
while (offset < buf.length) {
|
|
2167
|
+
const n = this._writeSync(this.fd, buf, offset, buf.length - offset);
|
|
2168
|
+
if (n <= 0)
|
|
2169
|
+
throw new Error(`access-log writeSync returned ${n}`);
|
|
2170
|
+
offset += n;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2535
2173
|
}
|
|
2536
|
-
function
|
|
2537
|
-
const
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
const
|
|
2541
|
-
const
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2174
|
+
function rfc3339Utc() {
|
|
2175
|
+
const d = new Date;
|
|
2176
|
+
const yyyy = d.getUTCFullYear().toString().padStart(4, "0");
|
|
2177
|
+
const mm = (d.getUTCMonth() + 1).toString().padStart(2, "0");
|
|
2178
|
+
const dd = d.getUTCDate().toString().padStart(2, "0");
|
|
2179
|
+
const hh = d.getUTCHours().toString().padStart(2, "0");
|
|
2180
|
+
const mi = d.getUTCMinutes().toString().padStart(2, "0");
|
|
2181
|
+
const ss = d.getUTCSeconds().toString().padStart(2, "0");
|
|
2182
|
+
const ms = d.getUTCMilliseconds().toString().padStart(3, "0");
|
|
2183
|
+
return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}Z`;
|
|
2545
2184
|
}
|
|
2546
|
-
function
|
|
2547
|
-
|
|
2548
|
-
if (!bearerMatch)
|
|
2549
|
-
return false;
|
|
2550
|
-
const params = bearerMatch[1];
|
|
2551
|
-
const match = params.match(/use_id_token_as_bearer="([^"]+)"/);
|
|
2552
|
-
if (!match)
|
|
2553
|
-
return false;
|
|
2554
|
-
return match[1] === "true";
|
|
2185
|
+
function base64(bytes) {
|
|
2186
|
+
return Buffer.from(bytes).toString("base64");
|
|
2555
2187
|
}
|
|
2556
|
-
function
|
|
2557
|
-
|
|
2558
|
-
if (!bearerMatch)
|
|
2559
|
-
return null;
|
|
2560
|
-
const params = bearerMatch[1];
|
|
2561
|
-
const match = params.match(/device_code_client_id="([^"]+)"/);
|
|
2562
|
-
if (!match)
|
|
2563
|
-
return null;
|
|
2564
|
-
return match[1];
|
|
2188
|
+
function roundTo2(f) {
|
|
2189
|
+
return Math.round(f * 100) / 100;
|
|
2565
2190
|
}
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2191
|
+
|
|
2192
|
+
class AccessLogHook {
|
|
2193
|
+
sink;
|
|
2194
|
+
serverVersion;
|
|
2195
|
+
level;
|
|
2196
|
+
constructor(sink, options = {}) {
|
|
2197
|
+
this.sink = sink;
|
|
2198
|
+
if (typeof options === "string") {
|
|
2199
|
+
this.serverVersion = options;
|
|
2200
|
+
this.level = "INFO";
|
|
2201
|
+
} else {
|
|
2202
|
+
this.serverVersion = options.serverVersion ?? "";
|
|
2203
|
+
this.level = options.level ?? "INFO";
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
onDispatchStart(_info) {
|
|
2207
|
+
const token = { startNs: process.hrtime.bigint() };
|
|
2208
|
+
return token;
|
|
2209
|
+
}
|
|
2210
|
+
onDispatchEnd(token, info, stats, error) {
|
|
2211
|
+
const t = token;
|
|
2212
|
+
const durationMs = t ? roundTo2(Number(process.hrtime.bigint() - t.startNs) / 1e6) : 0;
|
|
2213
|
+
const status = error ? "error" : "ok";
|
|
2214
|
+
const errType = error ? error.type ?? error.constructor.name : "";
|
|
2215
|
+
const errMsg = error?.message ?? "";
|
|
2216
|
+
const protocol = info.protocol ?? "";
|
|
2217
|
+
const rec = {
|
|
2218
|
+
timestamp: rfc3339Utc(),
|
|
2219
|
+
level: "INFO",
|
|
2220
|
+
logger: "vgi_rpc.access",
|
|
2221
|
+
message: `${protocol}.${info.method} ${status}`,
|
|
2222
|
+
server_id: info.serverId,
|
|
2223
|
+
protocol,
|
|
2224
|
+
protocol_hash: info.protocolHash ?? "",
|
|
2225
|
+
method: info.method,
|
|
2226
|
+
method_type: info.methodType,
|
|
2227
|
+
principal: info.principal ?? "",
|
|
2228
|
+
auth_domain: info.authDomain ?? "",
|
|
2229
|
+
authenticated: info.authenticated ?? false,
|
|
2230
|
+
remote_addr: info.remoteAddr ?? "",
|
|
2231
|
+
duration_ms: durationMs,
|
|
2232
|
+
status,
|
|
2233
|
+
error_type: errType
|
|
2234
|
+
};
|
|
2235
|
+
if (errMsg)
|
|
2236
|
+
rec.error_message = errMsg;
|
|
2237
|
+
if (this.serverVersion)
|
|
2238
|
+
rec.server_version = this.serverVersion;
|
|
2239
|
+
if (info.protocolVersion)
|
|
2240
|
+
rec.protocol_version = info.protocolVersion;
|
|
2241
|
+
if (info.requestId)
|
|
2242
|
+
rec.request_id = info.requestId;
|
|
2243
|
+
if (info.requestData && info.requestData.length > 0) {
|
|
2244
|
+
const encoded = base64(info.requestData);
|
|
2245
|
+
if (this.level === "DEBUG") {
|
|
2246
|
+
rec.request_data = encoded;
|
|
2247
|
+
} else {
|
|
2248
|
+
rec.original_request_bytes = encoded.length;
|
|
2249
|
+
rec.truncated = true;
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
2252
|
+
if (info.methodType === "stream") {
|
|
2253
|
+
rec.stream_id = info.streamId ?? "00000000000000000000000000000000";
|
|
2254
|
+
}
|
|
2255
|
+
if (info.cancelled)
|
|
2256
|
+
rec.cancelled = true;
|
|
2257
|
+
if (stats.inputBatches + stats.outputBatches + stats.inputRows + stats.outputRows + stats.inputBytes + stats.outputBytes !== 0) {
|
|
2258
|
+
rec.input_batches = stats.inputBatches;
|
|
2259
|
+
rec.output_batches = stats.outputBatches;
|
|
2260
|
+
rec.input_rows = stats.inputRows;
|
|
2261
|
+
rec.output_rows = stats.outputRows;
|
|
2262
|
+
rec.input_bytes = stats.inputBytes;
|
|
2263
|
+
rec.output_bytes = stats.outputBytes;
|
|
2264
|
+
}
|
|
2265
|
+
try {
|
|
2266
|
+
this.sink.write(`${JSON.stringify(rec)}
|
|
2267
|
+
`);
|
|
2268
|
+
} catch {}
|
|
2269
|
+
}
|
|
2575
2270
|
}
|
|
2576
|
-
// src/
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
writer;
|
|
2588
|
-
writeFn;
|
|
2589
|
-
closed = false;
|
|
2590
|
-
constructor(writeFn, schema2) {
|
|
2591
|
-
this.writeFn = writeFn;
|
|
2592
|
-
this.writer = new RecordBatchStreamWriter2;
|
|
2593
|
-
this.writer.reset(undefined, schema2);
|
|
2594
|
-
this.drain();
|
|
2271
|
+
// src/auth.ts
|
|
2272
|
+
class AuthContext {
|
|
2273
|
+
domain;
|
|
2274
|
+
authenticated;
|
|
2275
|
+
principal;
|
|
2276
|
+
claims;
|
|
2277
|
+
constructor(domain, authenticated, principal, claims = {}) {
|
|
2278
|
+
this.domain = domain;
|
|
2279
|
+
this.authenticated = authenticated;
|
|
2280
|
+
this.principal = principal;
|
|
2281
|
+
this.claims = claims;
|
|
2595
2282
|
}
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
throw new Error("PipeIncrementalWriter already closed");
|
|
2599
|
-
this.writer._writeRecordBatch(batch);
|
|
2600
|
-
this.drain();
|
|
2283
|
+
static anonymous() {
|
|
2284
|
+
return new AuthContext("", false, null);
|
|
2601
2285
|
}
|
|
2602
|
-
|
|
2603
|
-
if (this.
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
const eos = new Uint8Array(new Int32Array([-1, 0]).buffer);
|
|
2607
|
-
this.writeFn(eos);
|
|
2286
|
+
requireAuthenticated() {
|
|
2287
|
+
if (!this.authenticated) {
|
|
2288
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2289
|
+
}
|
|
2608
2290
|
}
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2291
|
+
}
|
|
2292
|
+
// src/client/capabilities.ts
|
|
2293
|
+
var MAX_REQUEST_BYTES_HEADER = "VGI-Max-Request-Bytes";
|
|
2294
|
+
var UPLOAD_URL_HEADER = "VGI-Upload-URL-Support";
|
|
2295
|
+
var MAX_UPLOAD_BYTES_HEADER = "VGI-Max-Upload-Bytes";
|
|
2296
|
+
function parseHeaderInt(headers, name) {
|
|
2297
|
+
const raw = headers.get(name) ?? headers.get(name.toLowerCase());
|
|
2298
|
+
if (raw == null)
|
|
2299
|
+
return null;
|
|
2300
|
+
const parsed = Number.parseInt(raw, 10);
|
|
2301
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
2302
|
+
}
|
|
2303
|
+
function parseCapabilitiesFromHeaders(headers) {
|
|
2304
|
+
const uploadRaw = headers.get(UPLOAD_URL_HEADER) ?? headers.get(UPLOAD_URL_HEADER.toLowerCase());
|
|
2305
|
+
const uploadUrlSupport = uploadRaw === "true";
|
|
2306
|
+
let cacheExpiresAt = null;
|
|
2307
|
+
const cc = headers.get("Cache-Control") ?? headers.get("cache-control");
|
|
2308
|
+
if (cc) {
|
|
2309
|
+
for (const token of cc.split(",")) {
|
|
2310
|
+
const t = token.trim().toLowerCase();
|
|
2311
|
+
if (t.startsWith("max-age=")) {
|
|
2312
|
+
const seconds = Number.parseFloat(t.slice("max-age=".length));
|
|
2313
|
+
if (Number.isFinite(seconds)) {
|
|
2314
|
+
cacheExpiresAt = Date.now() + seconds * 1000;
|
|
2315
|
+
}
|
|
2316
|
+
break;
|
|
2317
|
+
}
|
|
2613
2318
|
}
|
|
2614
|
-
values.length = 0;
|
|
2615
2319
|
}
|
|
2320
|
+
return {
|
|
2321
|
+
maxRequestBytes: parseHeaderInt(headers, MAX_REQUEST_BYTES_HEADER),
|
|
2322
|
+
uploadUrlSupport,
|
|
2323
|
+
maxUploadBytes: parseHeaderInt(headers, MAX_UPLOAD_BYTES_HEADER),
|
|
2324
|
+
cacheExpiresAt
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
function isCapabilitySnapshotFresh(snapshot) {
|
|
2328
|
+
if (!snapshot)
|
|
2329
|
+
return false;
|
|
2330
|
+
if (snapshot.cacheExpiresAt == null)
|
|
2331
|
+
return true;
|
|
2332
|
+
return Date.now() < snapshot.cacheExpiresAt;
|
|
2616
2333
|
}
|
|
2617
2334
|
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2335
|
+
// src/client/stream.ts
|
|
2336
|
+
import { Field as Field2, makeData as makeData2, RecordBatch as RecordBatch2, Schema as Schema2, Struct as Struct2, vectorFromArray as vectorFromArray2 } from "@query-farm/apache-arrow";
|
|
2337
|
+
class HttpStreamSession {
|
|
2338
|
+
_baseUrl;
|
|
2339
|
+
_prefix;
|
|
2340
|
+
_method;
|
|
2341
|
+
_stateToken;
|
|
2342
|
+
_outputSchema;
|
|
2343
|
+
_inputSchema;
|
|
2621
2344
|
_onLog;
|
|
2345
|
+
_pendingBatches;
|
|
2346
|
+
_finished;
|
|
2622
2347
|
_header;
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
_outputSchema;
|
|
2628
|
-
_releaseBusy;
|
|
2629
|
-
_setDrainPromise;
|
|
2348
|
+
_compressionLevel;
|
|
2349
|
+
_compressFn;
|
|
2350
|
+
_decompressFn;
|
|
2351
|
+
_authorization;
|
|
2630
2352
|
_externalConfig;
|
|
2353
|
+
_postFn;
|
|
2631
2354
|
constructor(opts) {
|
|
2632
|
-
this.
|
|
2633
|
-
this.
|
|
2355
|
+
this._baseUrl = opts.baseUrl;
|
|
2356
|
+
this._prefix = opts.prefix;
|
|
2357
|
+
this._method = opts.method;
|
|
2358
|
+
this._stateToken = opts.stateToken;
|
|
2359
|
+
this._outputSchema = opts.outputSchema;
|
|
2360
|
+
this._inputSchema = opts.inputSchema;
|
|
2634
2361
|
this._onLog = opts.onLog;
|
|
2362
|
+
this._pendingBatches = opts.pendingBatches;
|
|
2363
|
+
this._finished = opts.finished;
|
|
2635
2364
|
this._header = opts.header;
|
|
2636
|
-
this.
|
|
2637
|
-
this.
|
|
2638
|
-
this.
|
|
2365
|
+
this._compressionLevel = opts.compressionLevel;
|
|
2366
|
+
this._compressFn = opts.compressFn;
|
|
2367
|
+
this._decompressFn = opts.decompressFn;
|
|
2368
|
+
this._authorization = opts.authorization;
|
|
2639
2369
|
this._externalConfig = opts.externalConfig;
|
|
2370
|
+
this._postFn = opts.postFn;
|
|
2371
|
+
}
|
|
2372
|
+
async _post(url, body) {
|
|
2373
|
+
if (this._postFn)
|
|
2374
|
+
return this._postFn(url, body);
|
|
2375
|
+
return fetch(url, {
|
|
2376
|
+
method: "POST",
|
|
2377
|
+
headers: this._buildHeaders(),
|
|
2378
|
+
body: await this._prepareBody(body)
|
|
2379
|
+
});
|
|
2640
2380
|
}
|
|
2641
2381
|
get header() {
|
|
2642
2382
|
return this._header;
|
|
2643
2383
|
}
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2384
|
+
_buildHeaders() {
|
|
2385
|
+
const headers = {
|
|
2386
|
+
"Content-Type": ARROW_CONTENT_TYPE
|
|
2387
|
+
};
|
|
2388
|
+
if (this._compressionLevel != null && this._compressFn) {
|
|
2389
|
+
headers["Content-Encoding"] = "zstd";
|
|
2390
|
+
}
|
|
2391
|
+
if (this._compressionLevel != null && this._decompressFn) {
|
|
2392
|
+
headers["Accept-Encoding"] = "zstd";
|
|
2393
|
+
}
|
|
2394
|
+
if (this._authorization) {
|
|
2395
|
+
headers.Authorization = this._authorization;
|
|
2396
|
+
}
|
|
2397
|
+
return headers;
|
|
2398
|
+
}
|
|
2399
|
+
async _prepareBody(content) {
|
|
2400
|
+
if (this._compressionLevel != null && this._compressFn) {
|
|
2401
|
+
return await this._compressFn(content, this._compressionLevel);
|
|
2402
|
+
}
|
|
2403
|
+
return content;
|
|
2404
|
+
}
|
|
2405
|
+
async _readResponse(resp) {
|
|
2406
|
+
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2407
|
+
if (resp.headers.get("Content-Encoding") === "zstd" && this._decompressFn) {
|
|
2408
|
+
body = new Uint8Array(await this._decompressFn(body));
|
|
2409
|
+
}
|
|
2410
|
+
return body;
|
|
2411
|
+
}
|
|
2412
|
+
async exchange(input) {
|
|
2413
|
+
if (this._stateToken === null) {
|
|
2414
|
+
throw new RpcError("ProtocolError", "Stream has finished — no state token available", "");
|
|
2415
|
+
}
|
|
2416
|
+
if (input.length === 0) {
|
|
2417
|
+
const zeroSchema = this._inputSchema ?? this._outputSchema;
|
|
2418
|
+
const emptyBatch = this._buildEmptyBatch(zeroSchema);
|
|
2419
|
+
const metadata2 = new Map;
|
|
2420
|
+
metadata2.set(STATE_KEY, this._stateToken);
|
|
2421
|
+
const batchWithMeta = new RecordBatch2(zeroSchema, emptyBatch.data, metadata2);
|
|
2422
|
+
return this._doExchange(zeroSchema, [batchWithMeta]);
|
|
2423
|
+
}
|
|
2424
|
+
const keys = Object.keys(input[0]);
|
|
2425
|
+
const fields = keys.map((key) => {
|
|
2426
|
+
let sample;
|
|
2427
|
+
for (const row of input) {
|
|
2428
|
+
if (row[key] != null) {
|
|
2429
|
+
sample = row[key];
|
|
2430
|
+
break;
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
const arrowType = inferArrowType(sample);
|
|
2434
|
+
const nullable = input.some((row) => row[key] == null);
|
|
2435
|
+
return new Field2(key, arrowType, nullable);
|
|
2436
|
+
});
|
|
2437
|
+
const inputSchema = new Schema2(fields);
|
|
2438
|
+
const children = inputSchema.fields.map((f) => {
|
|
2439
|
+
const values = input.map((row) => row[f.name]);
|
|
2440
|
+
return vectorFromArray2(values, f.type).data[0];
|
|
2441
|
+
});
|
|
2442
|
+
const structType = new Struct2(inputSchema.fields);
|
|
2443
|
+
const data = makeData2({
|
|
2444
|
+
type: structType,
|
|
2445
|
+
length: input.length,
|
|
2446
|
+
children,
|
|
2447
|
+
nullCount: 0
|
|
2448
|
+
});
|
|
2449
|
+
const metadata = new Map;
|
|
2450
|
+
metadata.set(STATE_KEY, this._stateToken);
|
|
2451
|
+
const batch = new RecordBatch2(inputSchema, data, metadata);
|
|
2452
|
+
return this._doExchange(inputSchema, [batch]);
|
|
2453
|
+
}
|
|
2454
|
+
async _doExchange(schema2, batches) {
|
|
2455
|
+
const body = serializeIpcStream(schema2, batches);
|
|
2456
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
2457
|
+
if (resp.status === 401) {
|
|
2458
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2459
|
+
}
|
|
2460
|
+
const responseBody = await this._readResponse(resp);
|
|
2461
|
+
const { batches: responseBatches } = await readResponseBatches(responseBody);
|
|
2462
|
+
let resultRows = [];
|
|
2463
|
+
for (const batch of responseBatches) {
|
|
2464
|
+
if (batch.numRows === 0) {
|
|
2465
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2466
|
+
const token2 = batch.metadata?.get(STATE_KEY);
|
|
2467
|
+
if (token2) {
|
|
2468
|
+
this._stateToken = token2;
|
|
2469
|
+
}
|
|
2470
|
+
continue;
|
|
2471
|
+
}
|
|
2472
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2473
|
+
if (token) {
|
|
2474
|
+
this._stateToken = token;
|
|
2475
|
+
}
|
|
2476
|
+
resultRows = extractBatchRows(batch);
|
|
2477
|
+
}
|
|
2478
|
+
return resultRows;
|
|
2479
|
+
}
|
|
2480
|
+
_buildEmptyBatch(schema2) {
|
|
2481
|
+
const children = schema2.fields.map((f) => {
|
|
2482
|
+
return makeData2({ type: f.type, length: 0, nullCount: 0 });
|
|
2483
|
+
});
|
|
2484
|
+
const structType = new Struct2(schema2.fields);
|
|
2485
|
+
const data = makeData2({
|
|
2486
|
+
type: structType,
|
|
2487
|
+
length: 0,
|
|
2488
|
+
children,
|
|
2489
|
+
nullCount: 0
|
|
2490
|
+
});
|
|
2491
|
+
return new RecordBatch2(schema2, data);
|
|
2492
|
+
}
|
|
2493
|
+
async* [Symbol.asyncIterator]() {
|
|
2494
|
+
for (let batch of this._pendingBatches) {
|
|
2649
2495
|
if (batch.numRows === 0) {
|
|
2650
2496
|
if (isExternalLocationBatch(batch)) {
|
|
2651
|
-
|
|
2652
|
-
}
|
|
2653
|
-
|
|
2497
|
+
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2498
|
+
} else {
|
|
2499
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2654
2500
|
continue;
|
|
2655
2501
|
}
|
|
2656
2502
|
}
|
|
2657
|
-
|
|
2503
|
+
yield extractBatchRows(batch);
|
|
2658
2504
|
}
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
if (this._outputStreamOpened)
|
|
2505
|
+
this._pendingBatches = [];
|
|
2506
|
+
if (this._finished)
|
|
2662
2507
|
return;
|
|
2663
|
-
this.
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2508
|
+
if (this._stateToken === null)
|
|
2509
|
+
return;
|
|
2510
|
+
while (true) {
|
|
2511
|
+
const stateToken = this._stateToken;
|
|
2512
|
+
if (stateToken === null)
|
|
2513
|
+
return;
|
|
2514
|
+
const responseBody = await this._sendContinuation(stateToken);
|
|
2515
|
+
const { batches } = await readResponseBatches(responseBody);
|
|
2516
|
+
let gotContinuation = false;
|
|
2517
|
+
for (let batch of batches) {
|
|
2518
|
+
if (batch.numRows === 0) {
|
|
2519
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2520
|
+
if (token) {
|
|
2521
|
+
this._stateToken = token;
|
|
2522
|
+
gotContinuation = true;
|
|
2523
|
+
continue;
|
|
2524
|
+
}
|
|
2525
|
+
if (isExternalLocationBatch(batch)) {
|
|
2526
|
+
batch = await resolveExternalLocation(batch, this._externalConfig);
|
|
2527
|
+
} else {
|
|
2528
|
+
dispatchLogOrError(batch, this._onLog);
|
|
2529
|
+
continue;
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
yield extractBatchRows(batch);
|
|
2533
|
+
}
|
|
2534
|
+
if (!gotContinuation)
|
|
2535
|
+
break;
|
|
2667
2536
|
}
|
|
2668
2537
|
}
|
|
2669
|
-
async
|
|
2670
|
-
|
|
2671
|
-
|
|
2538
|
+
async _sendContinuation(token) {
|
|
2539
|
+
const emptySchema = new Schema2([]);
|
|
2540
|
+
const metadata = new Map;
|
|
2541
|
+
metadata.set(STATE_KEY, token);
|
|
2542
|
+
const structType = new Struct2(emptySchema.fields);
|
|
2543
|
+
const data = makeData2({
|
|
2544
|
+
type: structType,
|
|
2545
|
+
length: 1,
|
|
2546
|
+
children: [],
|
|
2547
|
+
nullCount: 0
|
|
2548
|
+
});
|
|
2549
|
+
const batch = new RecordBatch2(emptySchema, data, metadata);
|
|
2550
|
+
const body = serializeIpcStream(emptySchema, [batch]);
|
|
2551
|
+
const resp = await this._post(`${this._baseUrl}${this._prefix}/${this._method}/exchange`, body);
|
|
2552
|
+
if (resp.status === 401) {
|
|
2553
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2672
2554
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2555
|
+
return this._readResponse(resp);
|
|
2556
|
+
}
|
|
2557
|
+
close() {}
|
|
2558
|
+
}
|
|
2559
|
+
|
|
2560
|
+
// src/client/uploadUrl.ts
|
|
2561
|
+
import { Field as Field3, Int64 as Int642, RecordBatchReader as RecordBatchReader4, Schema as Schema3 } from "@query-farm/apache-arrow";
|
|
2562
|
+
var UPLOAD_URL_METHOD = "__upload_url__";
|
|
2563
|
+
var UPLOAD_URL_PARAMS_SCHEMA = new Schema3([new Field3("count", new Int642, false)]);
|
|
2564
|
+
async function requestUploadUrls(baseUrl, prefix, count, authorization) {
|
|
2565
|
+
const body = buildRequestIpc(UPLOAD_URL_PARAMS_SCHEMA, { count: BigInt(count) }, UPLOAD_URL_METHOD);
|
|
2566
|
+
const headers = { "Content-Type": ARROW_CONTENT_TYPE };
|
|
2567
|
+
if (authorization)
|
|
2568
|
+
headers.Authorization = authorization;
|
|
2569
|
+
const resp = await fetch(`${baseUrl}${prefix}/${UPLOAD_URL_METHOD}/init`, {
|
|
2570
|
+
method: "POST",
|
|
2571
|
+
headers,
|
|
2572
|
+
body
|
|
2573
|
+
});
|
|
2574
|
+
if (resp.status === 404) {
|
|
2575
|
+
throw new RpcError("NotSupported", "Server does not support upload URLs", "");
|
|
2576
|
+
}
|
|
2577
|
+
if (resp.status === 401) {
|
|
2578
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2579
|
+
}
|
|
2580
|
+
if (!resp.ok) {
|
|
2581
|
+
throw new RpcError("HttpError", `__upload_url__/init failed: HTTP ${resp.status}`, "");
|
|
2582
|
+
}
|
|
2583
|
+
const respBody = new Uint8Array(await resp.arrayBuffer());
|
|
2584
|
+
const reader = await RecordBatchReader4.from(respBody);
|
|
2585
|
+
await reader.open();
|
|
2586
|
+
const pairs = [];
|
|
2587
|
+
for (const batch of reader.readAll()) {
|
|
2588
|
+
if (batch.numRows === 0)
|
|
2589
|
+
continue;
|
|
2590
|
+
for (let r = 0;r < batch.numRows; r++) {
|
|
2591
|
+
const uploadUrl = batch.getChildAt(0)?.get(r);
|
|
2592
|
+
const downloadUrl = batch.getChildAt(1)?.get(r);
|
|
2593
|
+
const expiresRaw = batch.getChildAt(2)?.get(r);
|
|
2594
|
+
let expiresAt;
|
|
2595
|
+
if (expiresRaw instanceof Date) {
|
|
2596
|
+
expiresAt = expiresRaw;
|
|
2597
|
+
} else if (typeof expiresRaw === "bigint") {
|
|
2598
|
+
expiresAt = new Date(Number(expiresRaw / 1000n));
|
|
2599
|
+
} else if (typeof expiresRaw === "number") {
|
|
2600
|
+
expiresAt = new Date(expiresRaw);
|
|
2707
2601
|
} else {
|
|
2708
|
-
|
|
2602
|
+
expiresAt = new Date;
|
|
2709
2603
|
}
|
|
2710
|
-
|
|
2711
|
-
const values = input.map((row) => row[f.name]);
|
|
2712
|
-
return vectorFromArray2(values, f.type).data[0];
|
|
2713
|
-
});
|
|
2714
|
-
const structType = new Struct2(inputSchema.fields);
|
|
2715
|
-
const data = makeData2({
|
|
2716
|
-
type: structType,
|
|
2717
|
-
length: input.length,
|
|
2718
|
-
children,
|
|
2719
|
-
nullCount: 0
|
|
2720
|
-
});
|
|
2721
|
-
batch = new RecordBatch2(inputSchema, data);
|
|
2604
|
+
pairs.push({ uploadUrl, downloadUrl, expiresAt });
|
|
2722
2605
|
}
|
|
2723
|
-
|
|
2724
|
-
|
|
2606
|
+
}
|
|
2607
|
+
if (pairs.length === 0) {
|
|
2608
|
+
throw new RpcError("ProtocolError", "Server returned no upload URLs", "");
|
|
2609
|
+
}
|
|
2610
|
+
return pairs;
|
|
2611
|
+
}
|
|
2612
|
+
async function buildPointerRequestBody(originalBody, downloadUrl) {
|
|
2613
|
+
const reader = await RecordBatchReader4.from(originalBody);
|
|
2614
|
+
await reader.open();
|
|
2615
|
+
const schema2 = reader.schema;
|
|
2616
|
+
if (!schema2) {
|
|
2617
|
+
throw new RpcError("ProtocolError", "Original request body has no schema", "");
|
|
2618
|
+
}
|
|
2619
|
+
const batches = reader.readAll();
|
|
2620
|
+
if (batches.length === 0) {
|
|
2621
|
+
throw new RpcError("ProtocolError", "Original request body has no batches", "");
|
|
2622
|
+
}
|
|
2623
|
+
const original = batches[0];
|
|
2624
|
+
const originalMeta = original.metadata ?? new Map;
|
|
2625
|
+
const pointer = makeExternalLocationBatch(schema2, downloadUrl);
|
|
2626
|
+
const merged = new Map(pointer.metadata ?? new Map);
|
|
2627
|
+
const method = originalMeta.get(RPC_METHOD_KEY);
|
|
2628
|
+
const version = originalMeta.get(REQUEST_VERSION_KEY) ?? REQUEST_VERSION;
|
|
2629
|
+
if (method)
|
|
2630
|
+
merged.set(RPC_METHOD_KEY, method);
|
|
2631
|
+
merged.set(REQUEST_VERSION_KEY, version);
|
|
2632
|
+
for (const [k, v] of originalMeta) {
|
|
2633
|
+
if (!merged.has(k))
|
|
2634
|
+
merged.set(k, v);
|
|
2635
|
+
}
|
|
2636
|
+
const { RecordBatch: RecordBatch3 } = await import("@query-farm/apache-arrow");
|
|
2637
|
+
const pointerWithMeta = new RecordBatch3(schema2, pointer.data, merged);
|
|
2638
|
+
return serializeIpcStream(schema2, [pointerWithMeta]);
|
|
2639
|
+
}
|
|
2640
|
+
async function externalizeRequestBody(body, opts) {
|
|
2641
|
+
const pairs = await requestUploadUrls(opts.baseUrl, opts.prefix, 1, opts.authorization);
|
|
2642
|
+
const pair = pairs[0];
|
|
2643
|
+
if (opts.urlValidator) {
|
|
2644
|
+
opts.urlValidator(pair.uploadUrl);
|
|
2645
|
+
opts.urlValidator(pair.downloadUrl);
|
|
2646
|
+
}
|
|
2647
|
+
const putResp = await fetch(pair.uploadUrl, {
|
|
2648
|
+
method: "PUT",
|
|
2649
|
+
headers: { "Content-Type": ARROW_CONTENT_TYPE },
|
|
2650
|
+
body
|
|
2651
|
+
});
|
|
2652
|
+
if (!putResp.ok) {
|
|
2653
|
+
throw new RpcError("ExternalUploadFailed", `PUT to upload URL failed: HTTP ${putResp.status}`, "");
|
|
2654
|
+
}
|
|
2655
|
+
return buildPointerRequestBody(body, pair.downloadUrl);
|
|
2656
|
+
}
|
|
2657
|
+
|
|
2658
|
+
// src/client/connect.ts
|
|
2659
|
+
function httpConnect(baseUrl, options) {
|
|
2660
|
+
const prefix = (options?.prefix ?? "").replace(/\/+$/, "");
|
|
2661
|
+
const onLog = options?.onLog;
|
|
2662
|
+
const compressionLevel = options?.compressionLevel;
|
|
2663
|
+
const authorization = options?.authorization;
|
|
2664
|
+
const externalConfig = options?.externalLocation;
|
|
2665
|
+
let methodCache = null;
|
|
2666
|
+
let serverProtocolVersion = "";
|
|
2667
|
+
let compressFn;
|
|
2668
|
+
let decompressFn;
|
|
2669
|
+
let compressionLoaded = false;
|
|
2670
|
+
let capabilities = null;
|
|
2671
|
+
function updateCapabilitiesFromResponse(resp) {
|
|
2672
|
+
const next = parseCapabilitiesFromHeaders(resp.headers);
|
|
2673
|
+
if (next.maxRequestBytes != null || next.uploadUrlSupport) {
|
|
2674
|
+
capabilities = next;
|
|
2725
2675
|
}
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2676
|
+
}
|
|
2677
|
+
async function maybeExternalize(body) {
|
|
2678
|
+
const caps = isCapabilitySnapshotFresh(capabilities) ? capabilities : null;
|
|
2679
|
+
if (!caps)
|
|
2680
|
+
return body;
|
|
2681
|
+
if (!caps.uploadUrlSupport)
|
|
2682
|
+
return body;
|
|
2683
|
+
if (caps.maxRequestBytes == null || body.byteLength <= caps.maxRequestBytes)
|
|
2684
|
+
return body;
|
|
2685
|
+
return externalizeRequestBody(body, {
|
|
2686
|
+
baseUrl,
|
|
2687
|
+
prefix,
|
|
2688
|
+
authorization,
|
|
2689
|
+
urlValidator: externalConfig?.urlValidator ?? null
|
|
2690
|
+
});
|
|
2691
|
+
}
|
|
2692
|
+
async function postWithExternalization(url, body) {
|
|
2693
|
+
const sendBody = await maybeExternalize(body);
|
|
2694
|
+
let resp = await fetch(url, {
|
|
2695
|
+
method: "POST",
|
|
2696
|
+
headers: buildHeaders(),
|
|
2697
|
+
body: await prepareBody(sendBody)
|
|
2698
|
+
});
|
|
2699
|
+
updateCapabilitiesFromResponse(resp);
|
|
2700
|
+
if (resp.status === 413 && capabilities?.uploadUrlSupport && body.byteLength > 0) {
|
|
2701
|
+
const externalized = await externalizeRequestBody(body, {
|
|
2702
|
+
baseUrl,
|
|
2703
|
+
prefix,
|
|
2704
|
+
authorization,
|
|
2705
|
+
urlValidator: externalConfig?.urlValidator ?? null
|
|
2706
|
+
});
|
|
2707
|
+
resp = await fetch(url, {
|
|
2708
|
+
method: "POST",
|
|
2709
|
+
headers: buildHeaders(),
|
|
2710
|
+
body: await prepareBody(externalized)
|
|
2711
|
+
});
|
|
2712
|
+
updateCapabilitiesFromResponse(resp);
|
|
2737
2713
|
}
|
|
2714
|
+
return resp;
|
|
2738
2715
|
}
|
|
2739
|
-
async
|
|
2740
|
-
if (
|
|
2716
|
+
async function ensureCompression() {
|
|
2717
|
+
if (compressionLoaded || compressionLevel == null)
|
|
2741
2718
|
return;
|
|
2742
|
-
this._closed = true;
|
|
2743
|
-
if (this._inputWriter) {
|
|
2744
|
-
this._inputWriter.close();
|
|
2745
|
-
this._inputWriter = null;
|
|
2746
|
-
}
|
|
2747
2719
|
try {
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2720
|
+
const mod = await Promise.resolve().then(() => (init_zstd(), exports_zstd));
|
|
2721
|
+
compressFn = mod.zstdCompress;
|
|
2722
|
+
decompressFn = mod.zstdDecompress;
|
|
2751
2723
|
} catch {}
|
|
2752
|
-
|
|
2724
|
+
compressionLoaded = true;
|
|
2753
2725
|
}
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
|
|
2760
|
-
const structType = new Struct2(tickSchema.fields);
|
|
2761
|
-
const tickData = makeData2({
|
|
2762
|
-
type: structType,
|
|
2763
|
-
length: 0,
|
|
2764
|
-
children: [],
|
|
2765
|
-
nullCount: 0
|
|
2766
|
-
});
|
|
2767
|
-
const tickBatch = new RecordBatch2(tickSchema, tickData);
|
|
2768
|
-
while (true) {
|
|
2769
|
-
this._inputWriter.write(tickBatch);
|
|
2770
|
-
await this._ensureOutputStream();
|
|
2771
|
-
const outputBatch = await this._readOutputBatch();
|
|
2772
|
-
if (outputBatch === null) {
|
|
2773
|
-
break;
|
|
2774
|
-
}
|
|
2775
|
-
yield extractBatchRows(outputBatch);
|
|
2776
|
-
}
|
|
2777
|
-
} finally {
|
|
2778
|
-
if (this._inputWriter) {
|
|
2779
|
-
this._inputWriter.close();
|
|
2780
|
-
this._inputWriter = null;
|
|
2781
|
-
}
|
|
2782
|
-
try {
|
|
2783
|
-
if (this._outputStreamOpened) {
|
|
2784
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2785
|
-
}
|
|
2786
|
-
} catch {}
|
|
2787
|
-
this._closed = true;
|
|
2788
|
-
this._releaseBusy();
|
|
2726
|
+
function buildHeaders() {
|
|
2727
|
+
const headers = {
|
|
2728
|
+
"Content-Type": ARROW_CONTENT_TYPE
|
|
2729
|
+
};
|
|
2730
|
+
if (compressionLevel != null && compressFn) {
|
|
2731
|
+
headers["Content-Encoding"] = "zstd";
|
|
2789
2732
|
}
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
if (this._closed)
|
|
2793
|
-
return;
|
|
2794
|
-
this._closed = true;
|
|
2795
|
-
if (this._inputWriter) {
|
|
2796
|
-
this._inputWriter.close();
|
|
2797
|
-
this._inputWriter = null;
|
|
2798
|
-
} else {
|
|
2799
|
-
const emptySchema = new Schema3([]);
|
|
2800
|
-
const ipc = serializeIpcStream(emptySchema, []);
|
|
2801
|
-
this._writeFn(ipc);
|
|
2733
|
+
if (compressionLevel != null && decompressFn) {
|
|
2734
|
+
headers["Accept-Encoding"] = "zstd";
|
|
2802
2735
|
}
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
if (!this._outputStreamOpened) {
|
|
2806
|
-
const schema2 = await this._reader.openNextStream();
|
|
2807
|
-
if (schema2) {
|
|
2808
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2809
|
-
}
|
|
2810
|
-
} else {
|
|
2811
|
-
while (await this._reader.readNextBatch() !== null) {}
|
|
2812
|
-
}
|
|
2813
|
-
} catch {} finally {
|
|
2814
|
-
this._releaseBusy();
|
|
2815
|
-
}
|
|
2816
|
-
})();
|
|
2817
|
-
this._setDrainPromise(drainPromise);
|
|
2818
|
-
}
|
|
2819
|
-
}
|
|
2820
|
-
function pipeConnect(readable, writable, options) {
|
|
2821
|
-
const onLog = options?.onLog;
|
|
2822
|
-
const externalConfig = options?.externalLocation;
|
|
2823
|
-
let reader = null;
|
|
2824
|
-
let readerPromise = null;
|
|
2825
|
-
let methodCache = null;
|
|
2826
|
-
let protocolName = "";
|
|
2827
|
-
let serverProtocolVersion = "";
|
|
2828
|
-
let _busy = false;
|
|
2829
|
-
let _drainPromise = null;
|
|
2830
|
-
let closed = false;
|
|
2831
|
-
const writeFn = (bytes) => {
|
|
2832
|
-
writable.write(bytes);
|
|
2833
|
-
writable.flush?.();
|
|
2834
|
-
};
|
|
2835
|
-
async function ensureReader() {
|
|
2836
|
-
if (reader)
|
|
2837
|
-
return reader;
|
|
2838
|
-
if (!readerPromise) {
|
|
2839
|
-
readerPromise = IpcStreamReader.create(readable);
|
|
2736
|
+
if (authorization) {
|
|
2737
|
+
headers.Authorization = authorization;
|
|
2840
2738
|
}
|
|
2841
|
-
|
|
2842
|
-
return reader;
|
|
2739
|
+
return headers;
|
|
2843
2740
|
}
|
|
2844
|
-
async function
|
|
2845
|
-
if (
|
|
2846
|
-
await
|
|
2847
|
-
_drainPromise = null;
|
|
2848
|
-
}
|
|
2849
|
-
if (_busy) {
|
|
2850
|
-
throw new Error("Pipe transport is busy — another call or stream is in progress. " + "Pipe connections are single-threaded; wait for the current operation to complete.");
|
|
2741
|
+
async function prepareBody(content) {
|
|
2742
|
+
if (compressionLevel != null && compressFn) {
|
|
2743
|
+
return await compressFn(content, compressionLevel);
|
|
2851
2744
|
}
|
|
2852
|
-
|
|
2745
|
+
return content;
|
|
2853
2746
|
}
|
|
2854
|
-
function
|
|
2855
|
-
|
|
2747
|
+
function checkAuth(resp) {
|
|
2748
|
+
if (resp.status === 401) {
|
|
2749
|
+
throw new RpcError("AuthenticationError", "Authentication required", "");
|
|
2750
|
+
}
|
|
2856
2751
|
}
|
|
2857
|
-
function
|
|
2858
|
-
|
|
2752
|
+
async function readResponse(resp) {
|
|
2753
|
+
let body = new Uint8Array(await resp.arrayBuffer());
|
|
2754
|
+
if (resp.headers.get("Content-Encoding") === "zstd" && decompressFn) {
|
|
2755
|
+
body = new Uint8Array(await decompressFn(body));
|
|
2756
|
+
}
|
|
2757
|
+
return body;
|
|
2859
2758
|
}
|
|
2860
2759
|
async function ensureMethodCache() {
|
|
2861
2760
|
if (methodCache)
|
|
2862
2761
|
return methodCache;
|
|
2863
|
-
await
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
protocolName = desc.protocolName;
|
|
2875
|
-
serverProtocolVersion = desc.protocolVersion;
|
|
2876
|
-
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2877
|
-
return methodCache;
|
|
2878
|
-
} finally {
|
|
2879
|
-
releaseBusy();
|
|
2880
|
-
}
|
|
2762
|
+
await ensureCompression();
|
|
2763
|
+
const desc = await httpIntrospect(baseUrl, {
|
|
2764
|
+
prefix,
|
|
2765
|
+
authorization,
|
|
2766
|
+
compressionLevel,
|
|
2767
|
+
compressFn,
|
|
2768
|
+
decompressFn
|
|
2769
|
+
});
|
|
2770
|
+
methodCache = new Map(desc.methods.map((m) => [m.name, m]));
|
|
2771
|
+
serverProtocolVersion = desc.protocolVersion;
|
|
2772
|
+
return methodCache;
|
|
2881
2773
|
}
|
|
2882
2774
|
return {
|
|
2883
2775
|
async call(method, params) {
|
|
2776
|
+
await ensureCompression();
|
|
2884
2777
|
const methods = await ensureMethodCache();
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2778
|
+
const info = methods.get(method);
|
|
2779
|
+
if (!info) {
|
|
2780
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2781
|
+
}
|
|
2782
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2783
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2784
|
+
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}`, body);
|
|
2785
|
+
checkAuth(resp);
|
|
2786
|
+
const responseBody = await readResponse(resp);
|
|
2787
|
+
const { batches } = await readResponseBatches(responseBody);
|
|
2788
|
+
let resultBatch = null;
|
|
2789
|
+
for (let batch of batches) {
|
|
2790
|
+
if (batch.numRows === 0) {
|
|
2791
|
+
if (isExternalLocationBatch(batch)) {
|
|
2792
|
+
batch = await resolveExternalLocation(batch, externalConfig);
|
|
2793
|
+
} else {
|
|
2794
|
+
dispatchLogOrError(batch, onLog);
|
|
2795
|
+
continue;
|
|
2796
|
+
}
|
|
2890
2797
|
}
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2798
|
+
resultBatch = batch;
|
|
2799
|
+
}
|
|
2800
|
+
if (!resultBatch) {
|
|
2801
|
+
return null;
|
|
2802
|
+
}
|
|
2803
|
+
const rows = extractBatchRows(resultBatch);
|
|
2804
|
+
if (rows.length === 0)
|
|
2805
|
+
return null;
|
|
2806
|
+
const result = rows[0];
|
|
2807
|
+
if (info.resultSchema.fields.length === 0)
|
|
2808
|
+
return null;
|
|
2809
|
+
return result;
|
|
2810
|
+
},
|
|
2811
|
+
async stream(method, params) {
|
|
2812
|
+
await ensureCompression();
|
|
2813
|
+
const methods = await ensureMethodCache();
|
|
2814
|
+
const info = methods.get(method);
|
|
2815
|
+
if (!info) {
|
|
2816
|
+
throw new Error(`Unknown method: '${method}'`);
|
|
2817
|
+
}
|
|
2818
|
+
const fullParams = { ...info.defaults ?? {}, ...params ?? {} };
|
|
2819
|
+
const body = buildRequestIpc(info.paramsSchema, fullParams, method, { protocolVersion: serverProtocolVersion });
|
|
2820
|
+
const resp = await postWithExternalization(`${baseUrl}${prefix}/${method}/init`, body);
|
|
2821
|
+
checkAuth(resp);
|
|
2822
|
+
const responseBody = await readResponse(resp);
|
|
2823
|
+
let header = null;
|
|
2824
|
+
let stateToken = null;
|
|
2825
|
+
const pendingBatches = [];
|
|
2826
|
+
let finished = false;
|
|
2827
|
+
let streamSchema = null;
|
|
2828
|
+
if (info.headerSchema) {
|
|
2829
|
+
const reader = await readSequentialStreams(responseBody);
|
|
2830
|
+
const headerStream = await reader.readStream();
|
|
2831
|
+
if (headerStream) {
|
|
2832
|
+
for (const batch of headerStream.batches) {
|
|
2833
|
+
if (batch.numRows === 0) {
|
|
2834
|
+
dispatchLogOrError(batch, onLog);
|
|
2835
|
+
continue;
|
|
2836
|
+
}
|
|
2837
|
+
const rows = extractBatchRows(batch);
|
|
2838
|
+
if (rows.length > 0) {
|
|
2839
|
+
header = rows[0];
|
|
2840
|
+
}
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
const dataStream = await reader.readStream();
|
|
2844
|
+
if (dataStream) {
|
|
2845
|
+
streamSchema = dataStream.schema;
|
|
2898
2846
|
}
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
if (
|
|
2903
|
-
|
|
2904
|
-
|
|
2847
|
+
const headerErrorBatches = [];
|
|
2848
|
+
if (dataStream) {
|
|
2849
|
+
for (const batch of dataStream.batches) {
|
|
2850
|
+
if (batch.numRows === 0) {
|
|
2851
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2852
|
+
if (token) {
|
|
2853
|
+
stateToken = token;
|
|
2854
|
+
continue;
|
|
2855
|
+
}
|
|
2856
|
+
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2857
|
+
if (level === "EXCEPTION") {
|
|
2858
|
+
headerErrorBatches.push(batch);
|
|
2859
|
+
continue;
|
|
2860
|
+
}
|
|
2905
2861
|
dispatchLogOrError(batch, onLog);
|
|
2906
2862
|
continue;
|
|
2907
2863
|
}
|
|
2864
|
+
pendingBatches.push(batch);
|
|
2908
2865
|
}
|
|
2909
|
-
resultBatch = batch;
|
|
2910
2866
|
}
|
|
2911
|
-
if (
|
|
2912
|
-
|
|
2867
|
+
if (headerErrorBatches.length > 0) {
|
|
2868
|
+
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2869
|
+
pendingBatches.push(...headerErrorBatches);
|
|
2870
|
+
} else {
|
|
2871
|
+
for (const batch of headerErrorBatches) {
|
|
2872
|
+
dispatchLogOrError(batch, onLog);
|
|
2873
|
+
}
|
|
2874
|
+
}
|
|
2913
2875
|
}
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
return null;
|
|
2917
|
-
if (info.resultSchema.fields.length === 0)
|
|
2918
|
-
return null;
|
|
2919
|
-
return rows[0];
|
|
2920
|
-
} finally {
|
|
2921
|
-
releaseBusy();
|
|
2922
|
-
}
|
|
2923
|
-
},
|
|
2924
|
-
async stream(method, params) {
|
|
2925
|
-
const methods = await ensureMethodCache();
|
|
2926
|
-
await acquireBusy();
|
|
2927
|
-
try {
|
|
2928
|
-
const info = methods.get(method);
|
|
2929
|
-
if (!info) {
|
|
2930
|
-
throw new Error(`Unknown method: '${method}'`);
|
|
2876
|
+
if (!dataStream && !stateToken) {
|
|
2877
|
+
finished = true;
|
|
2931
2878
|
}
|
|
2932
|
-
|
|
2933
|
-
const
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
header = rows[0];
|
|
2948
|
-
}
|
|
2879
|
+
} else {
|
|
2880
|
+
const { schema: responseSchema, batches } = await readResponseBatches(responseBody);
|
|
2881
|
+
streamSchema = responseSchema;
|
|
2882
|
+
const errorBatches = [];
|
|
2883
|
+
for (const batch of batches) {
|
|
2884
|
+
if (batch.numRows === 0) {
|
|
2885
|
+
const token = batch.metadata?.get(STATE_KEY);
|
|
2886
|
+
if (token) {
|
|
2887
|
+
stateToken = token;
|
|
2888
|
+
continue;
|
|
2889
|
+
}
|
|
2890
|
+
const level = batch.metadata?.get(LOG_LEVEL_KEY);
|
|
2891
|
+
if (level === "EXCEPTION") {
|
|
2892
|
+
errorBatches.push(batch);
|
|
2893
|
+
continue;
|
|
2949
2894
|
}
|
|
2895
|
+
dispatchLogOrError(batch, onLog);
|
|
2896
|
+
continue;
|
|
2950
2897
|
}
|
|
2898
|
+
pendingBatches.push(batch);
|
|
2951
2899
|
}
|
|
2952
|
-
|
|
2953
|
-
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
try {
|
|
2965
|
-
const r = await ensureReader();
|
|
2966
|
-
const emptySchema = new Schema3([]);
|
|
2967
|
-
const ipc = serializeIpcStream(emptySchema, []);
|
|
2968
|
-
writeFn(ipc);
|
|
2969
|
-
const outStream = await r.readStream();
|
|
2970
|
-
} catch {}
|
|
2971
|
-
releaseBusy();
|
|
2972
|
-
throw e;
|
|
2900
|
+
if (errorBatches.length > 0) {
|
|
2901
|
+
if (pendingBatches.length > 0 || stateToken !== null) {
|
|
2902
|
+
pendingBatches.push(...errorBatches);
|
|
2903
|
+
} else {
|
|
2904
|
+
for (const batch of errorBatches) {
|
|
2905
|
+
dispatchLogOrError(batch, onLog);
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
}
|
|
2909
|
+
}
|
|
2910
|
+
if (pendingBatches.length === 0 && stateToken === null) {
|
|
2911
|
+
finished = true;
|
|
2973
2912
|
}
|
|
2913
|
+
const outputSchema = (streamSchema && streamSchema.fields.length > 0 ? streamSchema : null) ?? (pendingBatches.length > 0 ? pendingBatches[0].schema : null) ?? info.outputSchema ?? info.resultSchema;
|
|
2914
|
+
return new HttpStreamSession({
|
|
2915
|
+
baseUrl,
|
|
2916
|
+
prefix,
|
|
2917
|
+
method,
|
|
2918
|
+
stateToken,
|
|
2919
|
+
outputSchema,
|
|
2920
|
+
inputSchema: info.inputSchema,
|
|
2921
|
+
onLog,
|
|
2922
|
+
pendingBatches,
|
|
2923
|
+
finished,
|
|
2924
|
+
header,
|
|
2925
|
+
compressionLevel,
|
|
2926
|
+
compressFn,
|
|
2927
|
+
decompressFn,
|
|
2928
|
+
authorization,
|
|
2929
|
+
externalConfig,
|
|
2930
|
+
postFn: postWithExternalization
|
|
2931
|
+
});
|
|
2974
2932
|
},
|
|
2975
2933
|
async describe() {
|
|
2976
|
-
|
|
2977
|
-
return {
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2934
|
+
await ensureCompression();
|
|
2935
|
+
return httpIntrospect(baseUrl, {
|
|
2936
|
+
prefix,
|
|
2937
|
+
authorization,
|
|
2938
|
+
compressionLevel,
|
|
2939
|
+
compressFn,
|
|
2940
|
+
decompressFn
|
|
2941
|
+
});
|
|
2982
2942
|
},
|
|
2983
|
-
close() {
|
|
2984
|
-
if (closed)
|
|
2985
|
-
return;
|
|
2986
|
-
closed = true;
|
|
2987
|
-
writable.end();
|
|
2988
|
-
}
|
|
2943
|
+
close() {}
|
|
2989
2944
|
};
|
|
2990
2945
|
}
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
cwd: options?.cwd,
|
|
2997
|
-
env: options?.env ? { ...process.env, ...options.env } : undefined
|
|
2998
|
-
});
|
|
2999
|
-
const stdout = proc.stdout;
|
|
3000
|
-
const writable = {
|
|
3001
|
-
write(data) {
|
|
3002
|
-
proc.stdin.write(data);
|
|
3003
|
-
},
|
|
3004
|
-
flush() {
|
|
3005
|
-
proc.stdin.flush();
|
|
3006
|
-
},
|
|
3007
|
-
end() {
|
|
3008
|
-
proc.stdin.end();
|
|
3009
|
-
}
|
|
3010
|
-
};
|
|
3011
|
-
const client = pipeConnect(stdout, writable, {
|
|
3012
|
-
onLog: options?.onLog,
|
|
3013
|
-
externalLocation: options?.externalLocation
|
|
3014
|
-
});
|
|
3015
|
-
const originalClose = client.close;
|
|
3016
|
-
client.close = () => {
|
|
3017
|
-
originalClose.call(client);
|
|
3018
|
-
try {
|
|
3019
|
-
proc.kill();
|
|
3020
|
-
} catch {}
|
|
2946
|
+
// src/client/oauth.ts
|
|
2947
|
+
function parseMetadataJson(json) {
|
|
2948
|
+
const result = {
|
|
2949
|
+
resource: json.resource,
|
|
2950
|
+
authorizationServers: json.authorization_servers
|
|
3021
2951
|
};
|
|
3022
|
-
|
|
2952
|
+
if (json.scopes_supported)
|
|
2953
|
+
result.scopesSupported = json.scopes_supported;
|
|
2954
|
+
if (json.bearer_methods_supported)
|
|
2955
|
+
result.bearerMethodsSupported = json.bearer_methods_supported;
|
|
2956
|
+
if (json.resource_signing_alg_values_supported)
|
|
2957
|
+
result.resourceSigningAlgValuesSupported = json.resource_signing_alg_values_supported;
|
|
2958
|
+
if (json.resource_name)
|
|
2959
|
+
result.resourceName = json.resource_name;
|
|
2960
|
+
if (json.resource_documentation)
|
|
2961
|
+
result.resourceDocumentation = json.resource_documentation;
|
|
2962
|
+
if (json.resource_policy_uri)
|
|
2963
|
+
result.resourcePolicyUri = json.resource_policy_uri;
|
|
2964
|
+
if (json.resource_tos_uri)
|
|
2965
|
+
result.resourceTosUri = json.resource_tos_uri;
|
|
2966
|
+
if (json.client_id)
|
|
2967
|
+
result.clientId = json.client_id;
|
|
2968
|
+
if (json.client_secret)
|
|
2969
|
+
result.clientSecret = json.client_secret;
|
|
2970
|
+
if (json.use_id_token_as_bearer)
|
|
2971
|
+
result.useIdTokenAsBearer = json.use_id_token_as_bearer;
|
|
2972
|
+
if (json.device_code_client_id)
|
|
2973
|
+
result.deviceCodeClientId = json.device_code_client_id;
|
|
2974
|
+
if (json.device_code_client_secret)
|
|
2975
|
+
result.deviceCodeClientSecret = json.device_code_client_secret;
|
|
2976
|
+
return result;
|
|
2977
|
+
}
|
|
2978
|
+
async function httpOAuthMetadata(baseUrl, prefix) {
|
|
2979
|
+
const effectivePrefix = (prefix ?? "").replace(/\/+$/, "");
|
|
2980
|
+
const metadataUrl = `${baseUrl.replace(/\/+$/, "")}/.well-known/oauth-protected-resource${effectivePrefix}`;
|
|
2981
|
+
try {
|
|
2982
|
+
return await fetchOAuthMetadata(metadataUrl);
|
|
2983
|
+
} catch {
|
|
2984
|
+
return null;
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
async function fetchOAuthMetadata(metadataUrl) {
|
|
2988
|
+
const response = await fetch(metadataUrl);
|
|
2989
|
+
if (!response.ok) {
|
|
2990
|
+
throw new Error(`Failed to fetch OAuth metadata from ${metadataUrl}: ${response.status}`);
|
|
2991
|
+
}
|
|
2992
|
+
const json = await response.json();
|
|
2993
|
+
return parseMetadataJson(json);
|
|
2994
|
+
}
|
|
2995
|
+
function parseResourceMetadataUrl(wwwAuthenticate) {
|
|
2996
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
2997
|
+
if (!bearerMatch)
|
|
2998
|
+
return null;
|
|
2999
|
+
const params = bearerMatch[1];
|
|
3000
|
+
const metadataMatch = params.match(/resource_metadata="([^"]+)"/);
|
|
3001
|
+
if (!metadataMatch)
|
|
3002
|
+
return null;
|
|
3003
|
+
return metadataMatch[1];
|
|
3004
|
+
}
|
|
3005
|
+
function parseClientId(wwwAuthenticate) {
|
|
3006
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3007
|
+
if (!bearerMatch)
|
|
3008
|
+
return null;
|
|
3009
|
+
const params = bearerMatch[1];
|
|
3010
|
+
const clientIdMatch = params.match(/client_id="([^"]+)"/);
|
|
3011
|
+
if (!clientIdMatch)
|
|
3012
|
+
return null;
|
|
3013
|
+
return clientIdMatch[1];
|
|
3014
|
+
}
|
|
3015
|
+
function parseClientSecret(wwwAuthenticate) {
|
|
3016
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3017
|
+
if (!bearerMatch)
|
|
3018
|
+
return null;
|
|
3019
|
+
const params = bearerMatch[1];
|
|
3020
|
+
const match = params.match(/client_secret="([^"]+)"/);
|
|
3021
|
+
if (!match)
|
|
3022
|
+
return null;
|
|
3023
|
+
return match[1];
|
|
3024
|
+
}
|
|
3025
|
+
function parseUseIdTokenAsBearer(wwwAuthenticate) {
|
|
3026
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3027
|
+
if (!bearerMatch)
|
|
3028
|
+
return false;
|
|
3029
|
+
const params = bearerMatch[1];
|
|
3030
|
+
const match = params.match(/use_id_token_as_bearer="([^"]+)"/);
|
|
3031
|
+
if (!match)
|
|
3032
|
+
return false;
|
|
3033
|
+
return match[1] === "true";
|
|
3034
|
+
}
|
|
3035
|
+
function parseDeviceCodeClientId(wwwAuthenticate) {
|
|
3036
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3037
|
+
if (!bearerMatch)
|
|
3038
|
+
return null;
|
|
3039
|
+
const params = bearerMatch[1];
|
|
3040
|
+
const match = params.match(/device_code_client_id="([^"]+)"/);
|
|
3041
|
+
if (!match)
|
|
3042
|
+
return null;
|
|
3043
|
+
return match[1];
|
|
3044
|
+
}
|
|
3045
|
+
function parseDeviceCodeClientSecret(wwwAuthenticate) {
|
|
3046
|
+
const bearerMatch = wwwAuthenticate.match(/^Bearer\s+(.*)/i);
|
|
3047
|
+
if (!bearerMatch)
|
|
3048
|
+
return null;
|
|
3049
|
+
const params = bearerMatch[1];
|
|
3050
|
+
const match = params.match(/device_code_client_secret="([^"]+)"/);
|
|
3051
|
+
if (!match)
|
|
3052
|
+
return null;
|
|
3053
|
+
return match[1];
|
|
3023
3054
|
}
|
|
3024
3055
|
// src/http/auth.ts
|
|
3025
3056
|
function oauthResourceMetadataToJson(metadata) {
|
|
@@ -3304,6 +3335,7 @@ var TransportKind;
|
|
|
3304
3335
|
TransportKind2["PIPE"] = "pipe";
|
|
3305
3336
|
TransportKind2["HTTP"] = "http";
|
|
3306
3337
|
TransportKind2["UNIX"] = "unix";
|
|
3338
|
+
TransportKind2["TCP"] = "tcp";
|
|
3307
3339
|
})(TransportKind ||= {});
|
|
3308
3340
|
var EMPTY_COOKIES = new Map;
|
|
3309
3341
|
function cookieNotUnaryHttpError() {
|
|
@@ -8632,17 +8664,20 @@ class VgiRpcServer {
|
|
|
8632
8664
|
` + ` Direction: ${direction}`);
|
|
8633
8665
|
}
|
|
8634
8666
|
async run() {
|
|
8635
|
-
const stdin = process.stdin;
|
|
8636
8667
|
if (process.stdin.isTTY || process.stdout.isTTY) {
|
|
8637
8668
|
process.stderr.write("WARNING: This process communicates via Arrow IPC on stdin/stdout " + `and is not intended to be run interactively.
|
|
8638
8669
|
` + "It should be launched as a subprocess by an RPC client " + `(e.g. vgi_rpc.connect()).
|
|
8639
8670
|
`);
|
|
8640
8671
|
}
|
|
8641
|
-
const
|
|
8642
|
-
|
|
8672
|
+
const stdin = process.stdin;
|
|
8673
|
+
await this.serveConnection(stdin);
|
|
8674
|
+
}
|
|
8675
|
+
async serveConnection(readable, writable, transportKind = "pipe" /* PIPE */) {
|
|
8676
|
+
const reader = await IpcStreamReader.create(readable);
|
|
8677
|
+
const writer = new IpcStreamWriter(writable);
|
|
8643
8678
|
try {
|
|
8644
8679
|
while (true) {
|
|
8645
|
-
await this.notifyTransport(
|
|
8680
|
+
await this.notifyTransport(transportKind);
|
|
8646
8681
|
await this.serveOne(reader, writer);
|
|
8647
8682
|
}
|
|
8648
8683
|
} catch (e) {
|
|
@@ -8759,6 +8794,17 @@ class VgiRpcServer {
|
|
|
8759
8794
|
}
|
|
8760
8795
|
}
|
|
8761
8796
|
}
|
|
8797
|
+
// src/serve-stream.ts
|
|
8798
|
+
//! Serve a protocol over a caller-provided byte-stream pair — the stream
|
|
8799
|
+
//! sibling of `serveTcp` / `serveUnix`, with no socket/listener of its own.
|
|
8800
|
+
//!
|
|
8801
|
+
//! Useful for transports the launcher helpers don't cover: a Web Worker /
|
|
8802
|
+
//! `MessagePort` bridge (postMessage), an in-memory pipe, or a pre-connected
|
|
8803
|
+
//! socket. The host side already has this symmetry via `pipeConnect`.
|
|
8804
|
+
async function serveStream(protocol, options) {
|
|
8805
|
+
const server = new VgiRpcServer(protocol, options.serverOptions);
|
|
8806
|
+
await server.serveConnection(options.readable, options.writable, options.transportKind);
|
|
8807
|
+
}
|
|
8762
8808
|
// src/launcher/hash.ts
|
|
8763
8809
|
var HASH_LEN = 16;
|
|
8764
8810
|
function canonicalJson(value) {
|
|
@@ -9236,11 +9282,236 @@ function onceExit(proc) {
|
|
|
9236
9282
|
function delay(ms) {
|
|
9237
9283
|
return new Promise((r) => setTimeout(r, Math.max(0, ms)));
|
|
9238
9284
|
}
|
|
9285
|
+
// src/launcher/serve-tcp.ts
|
|
9286
|
+
import { createServer } from "node:net";
|
|
9287
|
+
var EMPTY_SCHEMA6 = schema([]);
|
|
9288
|
+
async function serveTcp(protocol, options = {}) {
|
|
9289
|
+
const host = options.host ?? "127.0.0.1";
|
|
9290
|
+
const requestedPort = options.port ?? 0;
|
|
9291
|
+
const idleTimeoutS = options.idleTimeout ?? 300;
|
|
9292
|
+
const startupGraceS = options.startupGraceSeconds ?? 5;
|
|
9293
|
+
const protocolVersion = options.protocolVersion ?? "";
|
|
9294
|
+
const serverId = options.serverId ?? crypto.randomUUID().replace(/-/g, "").slice(0, 12);
|
|
9295
|
+
const enableDescribe = options.enableDescribe ?? true;
|
|
9296
|
+
const dispatchHook = options.dispatchHook ?? null;
|
|
9297
|
+
const externalConfig = options.externalLocation;
|
|
9298
|
+
const onServeStart = options.onServeStart ?? null;
|
|
9299
|
+
const backlog = options.backlog ?? 128;
|
|
9300
|
+
const announcementSink = options.announcementSink ?? process.stdout;
|
|
9301
|
+
let describePromise = null;
|
|
9302
|
+
function describeInfo() {
|
|
9303
|
+
if (!describePromise) {
|
|
9304
|
+
describePromise = buildDescribeBatch(protocol.name, protocol.getMethods(), serverId).then(({ batch, metadata }) => ({
|
|
9305
|
+
batch,
|
|
9306
|
+
protocolHash: metadata.get("vgi_rpc.protocol_hash") ?? ""
|
|
9307
|
+
}));
|
|
9308
|
+
}
|
|
9309
|
+
return describePromise;
|
|
9310
|
+
}
|
|
9311
|
+
let serveStartFired = false;
|
|
9312
|
+
async function notifyTransport() {
|
|
9313
|
+
if (serveStartFired)
|
|
9314
|
+
return;
|
|
9315
|
+
if (onServeStart) {
|
|
9316
|
+
await onServeStart("tcp" /* TCP */);
|
|
9317
|
+
}
|
|
9318
|
+
serveStartFired = true;
|
|
9319
|
+
}
|
|
9320
|
+
const server = createServer({ allowHalfOpen: false });
|
|
9321
|
+
let activeConnections = 0;
|
|
9322
|
+
let idleTimer = null;
|
|
9323
|
+
let resolveDone = () => {};
|
|
9324
|
+
let rejectDone = () => {};
|
|
9325
|
+
const done = new Promise((resolve, reject) => {
|
|
9326
|
+
resolveDone = resolve;
|
|
9327
|
+
rejectDone = reject;
|
|
9328
|
+
});
|
|
9329
|
+
let stopped = false;
|
|
9330
|
+
function armIdleTimer() {
|
|
9331
|
+
if (idleTimeoutS <= 0)
|
|
9332
|
+
return;
|
|
9333
|
+
if (idleTimer)
|
|
9334
|
+
clearTimeout(idleTimer);
|
|
9335
|
+
idleTimer = setTimeout(() => {
|
|
9336
|
+
if (activeConnections === 0 && !stopped) {
|
|
9337
|
+
shutdown();
|
|
9338
|
+
}
|
|
9339
|
+
}, idleTimeoutS * 1000);
|
|
9340
|
+
}
|
|
9341
|
+
function disarmIdleTimer() {
|
|
9342
|
+
if (idleTimer) {
|
|
9343
|
+
clearTimeout(idleTimer);
|
|
9344
|
+
idleTimer = null;
|
|
9345
|
+
}
|
|
9346
|
+
}
|
|
9347
|
+
async function shutdown() {
|
|
9348
|
+
if (stopped)
|
|
9349
|
+
return;
|
|
9350
|
+
stopped = true;
|
|
9351
|
+
disarmIdleTimer();
|
|
9352
|
+
await new Promise((resolve) => {
|
|
9353
|
+
server.close(() => resolve());
|
|
9354
|
+
});
|
|
9355
|
+
resolveDone();
|
|
9356
|
+
}
|
|
9357
|
+
server.on("connection", (socket) => {
|
|
9358
|
+
try {
|
|
9359
|
+
socket.setNoDelay(true);
|
|
9360
|
+
} catch {}
|
|
9361
|
+
activeConnections += 1;
|
|
9362
|
+
disarmIdleTimer();
|
|
9363
|
+
handleConnection(socket).catch((err2) => {
|
|
9364
|
+
process.stderr.write(`vgi-rpc/tcp: connection failed: ${err2?.message ?? err2}
|
|
9365
|
+
`);
|
|
9366
|
+
}).finally(() => {
|
|
9367
|
+
activeConnections -= 1;
|
|
9368
|
+
socket.destroy();
|
|
9369
|
+
if (activeConnections === 0 && !stopped) {
|
|
9370
|
+
armIdleTimer();
|
|
9371
|
+
}
|
|
9372
|
+
});
|
|
9373
|
+
});
|
|
9374
|
+
server.on("error", (err2) => {
|
|
9375
|
+
if (stopped)
|
|
9376
|
+
return;
|
|
9377
|
+
rejectDone(err2);
|
|
9378
|
+
});
|
|
9379
|
+
async function handleConnection(socket) {
|
|
9380
|
+
const reader = await IpcStreamReader.create(socket);
|
|
9381
|
+
const writer = new IpcStreamWriter(socket);
|
|
9382
|
+
try {
|
|
9383
|
+
await notifyTransport();
|
|
9384
|
+
while (true) {
|
|
9385
|
+
try {
|
|
9386
|
+
await serveOnce(reader, writer);
|
|
9387
|
+
} catch (e) {
|
|
9388
|
+
const err2 = e;
|
|
9389
|
+
if (err2?.message?.includes("closed") || err2?.message?.includes("Expected Schema Message") || err2?.message?.includes("null or length 0") || err2?.message?.includes("EOF") || err2?.code === "EPIPE" || err2?.code === "ERR_STREAM_PREMATURE_CLOSE" || err2?.code === "ERR_STREAM_DESTROYED") {
|
|
9390
|
+
return;
|
|
9391
|
+
}
|
|
9392
|
+
throw e;
|
|
9393
|
+
}
|
|
9394
|
+
}
|
|
9395
|
+
} finally {
|
|
9396
|
+
try {
|
|
9397
|
+
await reader.cancel();
|
|
9398
|
+
} catch {}
|
|
9399
|
+
}
|
|
9400
|
+
}
|
|
9401
|
+
async function serveOnce(reader, writer) {
|
|
9402
|
+
const stream = await reader.readStream();
|
|
9403
|
+
if (!stream) {
|
|
9404
|
+
throw new Error("EOF");
|
|
9405
|
+
}
|
|
9406
|
+
const { schema: schema2, batches } = stream;
|
|
9407
|
+
if (batches.length === 0) {
|
|
9408
|
+
const err2 = new RpcError("ProtocolError", "Request stream contains no batches", "");
|
|
9409
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, err2, serverId, null);
|
|
9410
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9411
|
+
return;
|
|
9412
|
+
}
|
|
9413
|
+
const batch = batches[0];
|
|
9414
|
+
let methodName;
|
|
9415
|
+
let params;
|
|
9416
|
+
let requestId;
|
|
9417
|
+
try {
|
|
9418
|
+
const parsed = parseRequest(schema2, batch);
|
|
9419
|
+
methodName = parsed.methodName;
|
|
9420
|
+
params = parsed.params;
|
|
9421
|
+
requestId = parsed.requestId;
|
|
9422
|
+
} catch (e) {
|
|
9423
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, e, serverId, null);
|
|
9424
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9425
|
+
if (e instanceof VersionError || e instanceof RpcError)
|
|
9426
|
+
return;
|
|
9427
|
+
throw e;
|
|
9428
|
+
}
|
|
9429
|
+
if (methodName === DESCRIBE_METHOD_NAME && enableDescribe) {
|
|
9430
|
+
const { batch: descBatch } = await describeInfo();
|
|
9431
|
+
await writer.writeStream(descBatch.schema, [descBatch]);
|
|
9432
|
+
return;
|
|
9433
|
+
}
|
|
9434
|
+
const methods = protocol.getMethods();
|
|
9435
|
+
const method = methods.get(methodName);
|
|
9436
|
+
if (!method) {
|
|
9437
|
+
const available = [...methods.keys()].sort();
|
|
9438
|
+
const err2 = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
9439
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA6, err2, serverId, requestId);
|
|
9440
|
+
await writer.writeStream(EMPTY_SCHEMA6, [errBatch]);
|
|
9441
|
+
return;
|
|
9442
|
+
}
|
|
9443
|
+
const methodType = method.type === "unary" /* UNARY */ ? "unary" : "stream";
|
|
9444
|
+
let requestData;
|
|
9445
|
+
try {
|
|
9446
|
+
requestData = serializeBatch(batch);
|
|
9447
|
+
} catch {}
|
|
9448
|
+
const { protocolHash } = await describeInfo();
|
|
9449
|
+
const info = {
|
|
9450
|
+
method: methodName,
|
|
9451
|
+
methodType,
|
|
9452
|
+
serverId,
|
|
9453
|
+
requestId,
|
|
9454
|
+
protocol: protocol.name,
|
|
9455
|
+
protocolHash,
|
|
9456
|
+
protocolVersion,
|
|
9457
|
+
kind: "tcp" /* TCP */,
|
|
9458
|
+
principal: "",
|
|
9459
|
+
authDomain: "",
|
|
9460
|
+
authenticated: false,
|
|
9461
|
+
remoteAddr: "",
|
|
9462
|
+
requestData
|
|
9463
|
+
};
|
|
9464
|
+
const stats = {
|
|
9465
|
+
inputBatches: 0,
|
|
9466
|
+
outputBatches: 0,
|
|
9467
|
+
inputRows: 0,
|
|
9468
|
+
outputRows: 0,
|
|
9469
|
+
inputBytes: 0,
|
|
9470
|
+
outputBytes: 0
|
|
9471
|
+
};
|
|
9472
|
+
const token = dispatchHook?.onDispatchStart(info);
|
|
9473
|
+
let dispatchError;
|
|
9474
|
+
applyDefaults(params, method.defaults);
|
|
9475
|
+
try {
|
|
9476
|
+
if (method.type === "unary" /* UNARY */) {
|
|
9477
|
+
await dispatchUnary(method, params, writer, serverId, requestId, externalConfig, "tcp" /* TCP */);
|
|
9478
|
+
} else {
|
|
9479
|
+
await dispatchStream(method, params, writer, reader, serverId, requestId, externalConfig, "tcp" /* TCP */);
|
|
9480
|
+
}
|
|
9481
|
+
} catch (e) {
|
|
9482
|
+
dispatchError = e instanceof Error ? e : new Error(String(e));
|
|
9483
|
+
throw e;
|
|
9484
|
+
} finally {
|
|
9485
|
+
dispatchHook?.onDispatchEnd(token, info, stats, dispatchError);
|
|
9486
|
+
}
|
|
9487
|
+
}
|
|
9488
|
+
await new Promise((resolve, reject) => {
|
|
9489
|
+
server.listen({ host, port: requestedPort, backlog }, () => resolve());
|
|
9490
|
+
server.once("error", (err2) => reject(err2));
|
|
9491
|
+
});
|
|
9492
|
+
const address = server.address();
|
|
9493
|
+
const boundPort = typeof address === "object" && address ? address.port : requestedPort;
|
|
9494
|
+
options.onBound?.(host, boundPort);
|
|
9495
|
+
announcementSink.write(`TCP:${host}:${boundPort}
|
|
9496
|
+
`);
|
|
9497
|
+
if (idleTimeoutS > 0) {
|
|
9498
|
+
setTimeout(() => {
|
|
9499
|
+
if (activeConnections === 0 && !stopped)
|
|
9500
|
+
armIdleTimer();
|
|
9501
|
+
}, startupGraceS * 1000).unref?.();
|
|
9502
|
+
}
|
|
9503
|
+
return {
|
|
9504
|
+
host,
|
|
9505
|
+
port: boundPort,
|
|
9506
|
+
stop: shutdown,
|
|
9507
|
+
done
|
|
9508
|
+
};
|
|
9509
|
+
}
|
|
9239
9510
|
// src/launcher/serve-unix.ts
|
|
9240
9511
|
import { existsSync as existsSync2, unlinkSync as unlinkSync3 } from "node:fs";
|
|
9241
|
-
import { createServer } from "node:net";
|
|
9512
|
+
import { createServer as createServer2 } from "node:net";
|
|
9242
9513
|
import * as path2 from "node:path";
|
|
9243
|
-
var
|
|
9514
|
+
var EMPTY_SCHEMA7 = schema([]);
|
|
9244
9515
|
async function serveUnix(protocol, options) {
|
|
9245
9516
|
const sockPath = path2.resolve(options.unixPath);
|
|
9246
9517
|
const idleTimeoutS = options.idleTimeout ?? 300;
|
|
@@ -9277,7 +9548,7 @@ async function serveUnix(protocol, options) {
|
|
|
9277
9548
|
}
|
|
9278
9549
|
serveStartFired = true;
|
|
9279
9550
|
}
|
|
9280
|
-
const server =
|
|
9551
|
+
const server = createServer2({ allowHalfOpen: false });
|
|
9281
9552
|
let activeConnections = 0;
|
|
9282
9553
|
let idleTimer = null;
|
|
9283
9554
|
let resolveDone = () => {};
|
|
@@ -9366,8 +9637,8 @@ async function serveUnix(protocol, options) {
|
|
|
9366
9637
|
const { schema: schema2, batches } = stream;
|
|
9367
9638
|
if (batches.length === 0) {
|
|
9368
9639
|
const err2 = new RpcError("ProtocolError", "Request stream contains no batches", "");
|
|
9369
|
-
const errBatch = buildErrorBatch(
|
|
9370
|
-
await writer.writeStream(
|
|
9640
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, err2, serverId, null);
|
|
9641
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9371
9642
|
return;
|
|
9372
9643
|
}
|
|
9373
9644
|
const batch = batches[0];
|
|
@@ -9380,8 +9651,8 @@ async function serveUnix(protocol, options) {
|
|
|
9380
9651
|
params = parsed.params;
|
|
9381
9652
|
requestId = parsed.requestId;
|
|
9382
9653
|
} catch (e) {
|
|
9383
|
-
const errBatch = buildErrorBatch(
|
|
9384
|
-
await writer.writeStream(
|
|
9654
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, e, serverId, null);
|
|
9655
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9385
9656
|
if (e instanceof VersionError || e instanceof RpcError)
|
|
9386
9657
|
return;
|
|
9387
9658
|
throw e;
|
|
@@ -9396,8 +9667,8 @@ async function serveUnix(protocol, options) {
|
|
|
9396
9667
|
if (!method) {
|
|
9397
9668
|
const available = [...methods.keys()].sort();
|
|
9398
9669
|
const err2 = new Error(`Unknown method: '${methodName}'. Available methods: [${available.join(", ")}]`);
|
|
9399
|
-
const errBatch = buildErrorBatch(
|
|
9400
|
-
await writer.writeStream(
|
|
9670
|
+
const errBatch = buildErrorBatch(EMPTY_SCHEMA7, err2, serverId, requestId);
|
|
9671
|
+
await writer.writeStream(EMPTY_SCHEMA7, [errBatch]);
|
|
9401
9672
|
return;
|
|
9402
9673
|
}
|
|
9403
9674
|
const methodType = method.type === "unary" /* UNARY */ ? "unary" : "stream";
|
|
@@ -9476,11 +9747,14 @@ export {
|
|
|
9476
9747
|
uint162 as uint16,
|
|
9477
9748
|
tryAcquireLock,
|
|
9478
9749
|
toSchema,
|
|
9750
|
+
tcpConnect,
|
|
9479
9751
|
subprocessConnect,
|
|
9480
9752
|
str,
|
|
9481
9753
|
statusRows,
|
|
9482
9754
|
socketPaths,
|
|
9483
9755
|
serveUnix,
|
|
9756
|
+
serveTcp,
|
|
9757
|
+
serveStream,
|
|
9484
9758
|
resolveExternalLocation,
|
|
9485
9759
|
probeSocket,
|
|
9486
9760
|
pipeConnect,
|
|
@@ -9561,4 +9835,4 @@ export {
|
|
|
9561
9835
|
ARROW_CONTENT_TYPE
|
|
9562
9836
|
};
|
|
9563
9837
|
|
|
9564
|
-
//# debugId=
|
|
9838
|
+
//# debugId=FD35C475AB9ACBBD64756E2164756E21
|