@malloy-publisher/server 0.0.249 → 0.0.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +5 -0
- package/README.md +5 -0
- package/dist/app/api-doc.html +5 -0
- package/dist/app/api-doc.yaml +48 -10
- package/dist/app/assets/{EnvironmentPage-CB0NkVVs.js → EnvironmentPage-CiuWaYTk.js} +1 -1
- package/dist/app/assets/{HomePage-BEfG37L3.js → HomePage-ZXsmEAjw.js} +1 -1
- package/dist/app/assets/{LightMode--nzFqff1.js → LightMode-10iLdX5H.js} +1 -1
- package/dist/app/assets/{MainPage-owd1ua9R.js → MainPage-u-IAaBdE.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-BqaGRrxZ.js → MaterializationsPage-AcoL_Rnx.js} +1 -1
- package/dist/app/assets/{ModelPage-BydBAuRk.js → ModelPage-BjB-p4Qn.js} +1 -1
- package/dist/app/assets/{PackagePage-BDl_rs-E.js → PackagePage-le809jUO.js} +1 -1
- package/dist/app/assets/{RouteError-CyReblMg.js → RouteError-DTlqmNQd.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-zW7ndXQ8.js → ThemeEditorPage-DrmSzweH.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CzjOTRw8.js → WorkbookPage-CIRFZXlP.js} +1 -1
- package/dist/app/assets/{core-DFBwNhvb.es-BwyvPM0R.js → core-Cr-9Wkp3.es-YhIv13au.js} +1 -1
- package/dist/app/assets/{index-D6bZJBhc.js → index-CM5d1afN.js} +1 -1
- package/dist/app/assets/{index-Cmp8wg1s.js → index-CSFSsus4.js} +1 -1
- package/dist/app/assets/{index-t7IEDVsK.js → index-DP4bilgp.js} +5 -5
- package/dist/app/assets/{index-Byu4Enw7.js → index-DdUOWgW6.js} +1 -1
- package/dist/app/index.html +6 -1
- package/dist/runtime/publisher.js +3 -0
- package/dist/server.mjs +2635 -123
- package/package.json +3 -2
package/dist/server.mjs
CHANGED
|
@@ -248415,6 +248415,9 @@ function recordSourceBuildDuration(durationMs, engine) {
|
|
|
248415
248415
|
function recordDropTables(outcome, engine) {
|
|
248416
248416
|
dropTablesCounter().add(1, { outcome, engine });
|
|
248417
248417
|
}
|
|
248418
|
+
function recordStorageTableRetained(destination) {
|
|
248419
|
+
storageTableRetainedCounter().add(1, { destination });
|
|
248420
|
+
}
|
|
248418
248421
|
function recordStorageBuildFailure(destination, reason = "build_failed") {
|
|
248419
248422
|
storageBuildFailureCounter().add(1, { destination, reason });
|
|
248420
248423
|
}
|
|
@@ -248436,7 +248439,7 @@ function recordStorageServeRouting(outcome) {
|
|
|
248436
248439
|
function recordChainedStorageBuild(outcome) {
|
|
248437
248440
|
chainedStorageBuildCounter().add(1, { outcome });
|
|
248438
248441
|
}
|
|
248439
|
-
var resetHooks2, runCounter, runDuration, sourcesCounter, incrementalStepCounter, buildPlanComputeDuration, autoLoadCounter, connectionDigestSkipCounter, manifestBindCounter, manifestBindDegradedCounter, sourceBuildDuration, dropTablesCounter, scheduledFireCounter, storageServeRoutingCounter, storageBuildFailureCounter, attributionSkippedCounter, eligibilityRefusedCounter, serveShapeTierDropCounter, serveShapeTypeFallbackCounter, chainedStorageBuildCounter;
|
|
248442
|
+
var resetHooks2, runCounter, runDuration, sourcesCounter, incrementalStepCounter, buildPlanComputeDuration, autoLoadCounter, connectionDigestSkipCounter, manifestBindCounter, manifestBindDegradedCounter, sourceBuildDuration, dropTablesCounter, scheduledFireCounter, storageServeRoutingCounter, storageTableRetainedCounter, storageBuildFailureCounter, attributionSkippedCounter, eligibilityRefusedCounter, serveShapeTierDropCounter, serveShapeTypeFallbackCounter, chainedStorageBuildCounter;
|
|
248440
248443
|
var init_materialization_metrics = __esm(() => {
|
|
248441
248444
|
init_telemetry();
|
|
248442
248445
|
resetHooks2 = [];
|
|
@@ -248453,6 +248456,7 @@ var init_materialization_metrics = __esm(() => {
|
|
|
248453
248456
|
dropTablesCounter = lazyCounter2("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
|
|
248454
248457
|
scheduledFireCounter = lazyCounter2("publisher_materialization_scheduled_fires_total", "Standalone-scheduler attempts to fire a package's materialization.schedule. " + "Label: outcome ('fired'|'conflict'|'error').");
|
|
248455
248458
|
storageServeRoutingCounter = lazyCounter2("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|'runtime_live_fallback').");
|
|
248459
|
+
storageTableRetainedCounter = lazyCounter2("publisher_storage_tables_retained_total", "Tables a FAILED run left in a storage= destination and deliberately did not " + "reclaim, because the source is refreshed incrementally and the name may be " + "the one it serves from. Label: destination. Not all of these are orphans — " + "a rebuild at a fresh generational name is, a seed on the live serving name " + "is not, and the manifest entry cannot separate them — so read this as an " + "upper bound on what is accumulating rather than a leak count. It is the " + "only accounting there is until reclaiming a destination exists, which is " + "why it is a counter and not just a log line: the question is a rate, not " + "whether it ever happened.");
|
|
248456
248460
|
storageBuildFailureCounter = lazyCounter2("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Labels: destination (connection name), " + "reason ('build_failed'|'billed_read_not_captured'). The second is the " + "expensive one: the warehouse read ran and was charged, and the rows could " + "not be captured — so a re-drive pays for it again. Worth alerting on " + "separately from a failure that costs only a retry.");
|
|
248457
248461
|
attributionSkippedCounter = lazyCounter2("publisher_storage_build_attribution_skipped_total", "storage= builds whose warehouse read went out UNATTRIBUTED while tagging was " + "on. Label: reason ('job_listing_unavailable'|'tag_failed'|" + "'read_row_not_found'|'read_row_ambiguous'|'cost_query_failed'). " + "The read still ran and the " + "build still succeeded — what was lost is the label in the customer's own " + "query history, and the cost on this side. Without this an operator who " + "turns tagging on and sees nothing has a single log line to go on.");
|
|
248458
248462
|
eligibilityRefusedCounter = lazyCounter2("publisher_materialization_eligibility_refused_total", "storage= materialization-eligibility refusals. Label: reason " + "('free_parameter'|'given'|'authorize'|'not_duckdb_portable'|" + "'public_surface_unknown').");
|
|
@@ -250812,6 +250816,7 @@ var init_model = __esm(() => {
|
|
|
250812
250816
|
init_gate_registry_walk();
|
|
250813
250817
|
init_source_extraction();
|
|
250814
250818
|
init_authorize_metrics();
|
|
250819
|
+
init_path_safety();
|
|
250815
250820
|
MALLOY_VERSION = createRequire2(import.meta.url)("@malloydata/malloy/package.json").version;
|
|
250816
250821
|
Model = class Model {
|
|
250817
250822
|
packageName;
|
|
@@ -252518,7 +252523,12 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
|
|
|
252518
252523
|
};
|
|
252519
252524
|
}
|
|
252520
252525
|
static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
|
|
252521
|
-
|
|
252526
|
+
let fullModelPath;
|
|
252527
|
+
try {
|
|
252528
|
+
fullModelPath = safeJoinUnderRoot(packagePath, modelPath);
|
|
252529
|
+
} catch {
|
|
252530
|
+
throw new ModelNotFoundError(`${modelPath} does not exist.`);
|
|
252531
|
+
}
|
|
252522
252532
|
try {
|
|
252523
252533
|
if (!(await fs6.stat(fullModelPath)).isFile()) {
|
|
252524
252534
|
throw new ModelNotFoundError(`${modelPath} is not a file.`);
|
|
@@ -267987,6 +267997,1421 @@ var require_lunr = __commonJS((exports, module) => {
|
|
|
267987
267997
|
})();
|
|
267988
267998
|
});
|
|
267989
267999
|
|
|
268000
|
+
// ../../node_modules/ip-address/dist/address-error.js
|
|
268001
|
+
var require_address_error = __commonJS((exports) => {
|
|
268002
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268003
|
+
exports.AddressError = undefined;
|
|
268004
|
+
|
|
268005
|
+
class AddressError extends Error {
|
|
268006
|
+
constructor(message, parseMessage) {
|
|
268007
|
+
super(message);
|
|
268008
|
+
this.name = "AddressError";
|
|
268009
|
+
this.parseMessage = parseMessage;
|
|
268010
|
+
}
|
|
268011
|
+
}
|
|
268012
|
+
exports.AddressError = AddressError;
|
|
268013
|
+
});
|
|
268014
|
+
|
|
268015
|
+
// ../../node_modules/ip-address/dist/common.js
|
|
268016
|
+
var require_common8 = __commonJS((exports) => {
|
|
268017
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268018
|
+
exports.isInSubnet = isInSubnet;
|
|
268019
|
+
exports.isHostInSubnet = isHostInSubnet;
|
|
268020
|
+
exports.isCorrect = isCorrect;
|
|
268021
|
+
exports.prefixLengthFromMask = prefixLengthFromMask;
|
|
268022
|
+
exports.assertByteArray = assertByteArray;
|
|
268023
|
+
exports.numberToPaddedHex = numberToPaddedHex;
|
|
268024
|
+
exports.stringToPaddedHex = stringToPaddedHex;
|
|
268025
|
+
exports.testBit = testBit;
|
|
268026
|
+
var address_error_1 = require_address_error();
|
|
268027
|
+
function isInSubnet(address) {
|
|
268028
|
+
if (this.subnetMask < address.subnetMask) {
|
|
268029
|
+
return false;
|
|
268030
|
+
}
|
|
268031
|
+
return isHostInSubnet.call(this, address);
|
|
268032
|
+
}
|
|
268033
|
+
function isHostInSubnet(address) {
|
|
268034
|
+
return this.mask(address.subnetMask) === address.mask();
|
|
268035
|
+
}
|
|
268036
|
+
function isCorrect(defaultBits) {
|
|
268037
|
+
return function isCorrectForm() {
|
|
268038
|
+
if (this.addressMinusSuffix !== this.correctForm()) {
|
|
268039
|
+
return false;
|
|
268040
|
+
}
|
|
268041
|
+
if (this.subnetMask === defaultBits && !this.parsedSubnet) {
|
|
268042
|
+
return true;
|
|
268043
|
+
}
|
|
268044
|
+
return this.parsedSubnet === String(this.subnetMask);
|
|
268045
|
+
};
|
|
268046
|
+
}
|
|
268047
|
+
function prefixLengthFromMask(value, totalBits) {
|
|
268048
|
+
const binary = value.toString(2).padStart(totalBits, "0");
|
|
268049
|
+
if (binary.length > totalBits) {
|
|
268050
|
+
throw new address_error_1.AddressError("Invalid subnet mask.");
|
|
268051
|
+
}
|
|
268052
|
+
const firstZero = binary.indexOf("0");
|
|
268053
|
+
if (firstZero === -1) {
|
|
268054
|
+
return totalBits;
|
|
268055
|
+
}
|
|
268056
|
+
if (binary.slice(firstZero).includes("1")) {
|
|
268057
|
+
throw new address_error_1.AddressError("Invalid subnet mask.");
|
|
268058
|
+
}
|
|
268059
|
+
return firstZero;
|
|
268060
|
+
}
|
|
268061
|
+
function assertByteArray(bytes, byteCount, family, minimum) {
|
|
268062
|
+
if (bytes.length !== byteCount) {
|
|
268063
|
+
throw new address_error_1.AddressError(`${family} addresses require exactly ${byteCount} bytes`);
|
|
268064
|
+
}
|
|
268065
|
+
for (let i = 0;i < bytes.length; i++) {
|
|
268066
|
+
if (!Number.isInteger(bytes[i]) || bytes[i] < minimum || bytes[i] > 255) {
|
|
268067
|
+
throw new address_error_1.AddressError(`All bytes must be integers between ${minimum} and 255`);
|
|
268068
|
+
}
|
|
268069
|
+
}
|
|
268070
|
+
}
|
|
268071
|
+
function numberToPaddedHex(number2) {
|
|
268072
|
+
return number2.toString(16).padStart(2, "0");
|
|
268073
|
+
}
|
|
268074
|
+
function stringToPaddedHex(numberString) {
|
|
268075
|
+
return numberToPaddedHex(parseInt(numberString, 10));
|
|
268076
|
+
}
|
|
268077
|
+
function testBit(binaryValue, position) {
|
|
268078
|
+
const { length } = binaryValue;
|
|
268079
|
+
if (position > length) {
|
|
268080
|
+
return false;
|
|
268081
|
+
}
|
|
268082
|
+
const positionInString = length - position;
|
|
268083
|
+
return binaryValue.substring(positionInString, positionInString + 1) === "1";
|
|
268084
|
+
}
|
|
268085
|
+
});
|
|
268086
|
+
|
|
268087
|
+
// ../../node_modules/ip-address/dist/v4/constants.js
|
|
268088
|
+
var require_constants25 = __commonJS((exports) => {
|
|
268089
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268090
|
+
exports.RE_SUBNET_STRING = exports.RE_ADDRESS = exports.GROUPS = exports.BITS = undefined;
|
|
268091
|
+
exports.BITS = 32;
|
|
268092
|
+
exports.GROUPS = 4;
|
|
268093
|
+
exports.RE_ADDRESS = /^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$/g;
|
|
268094
|
+
exports.RE_SUBNET_STRING = /\/\d{1,2}$/;
|
|
268095
|
+
});
|
|
268096
|
+
|
|
268097
|
+
// ../../node_modules/ip-address/dist/ipv4.js
|
|
268098
|
+
var require_ipv4 = __commonJS((exports) => {
|
|
268099
|
+
var __createBinding = exports && exports.__createBinding || (Object.create ? function(o, m, k, k2) {
|
|
268100
|
+
if (k2 === undefined)
|
|
268101
|
+
k2 = k;
|
|
268102
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
268103
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
268104
|
+
desc = { enumerable: true, get: function() {
|
|
268105
|
+
return m[k];
|
|
268106
|
+
} };
|
|
268107
|
+
}
|
|
268108
|
+
Object.defineProperty(o, k2, desc);
|
|
268109
|
+
} : function(o, m, k, k2) {
|
|
268110
|
+
if (k2 === undefined)
|
|
268111
|
+
k2 = k;
|
|
268112
|
+
o[k2] = m[k];
|
|
268113
|
+
});
|
|
268114
|
+
var __setModuleDefault = exports && exports.__setModuleDefault || (Object.create ? function(o, v) {
|
|
268115
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
268116
|
+
} : function(o, v) {
|
|
268117
|
+
o["default"] = v;
|
|
268118
|
+
});
|
|
268119
|
+
var __importStar = exports && exports.__importStar || function(mod2) {
|
|
268120
|
+
if (mod2 && mod2.__esModule)
|
|
268121
|
+
return mod2;
|
|
268122
|
+
var result = {};
|
|
268123
|
+
if (mod2 != null) {
|
|
268124
|
+
for (var k in mod2)
|
|
268125
|
+
if (k !== "default" && Object.prototype.hasOwnProperty.call(mod2, k))
|
|
268126
|
+
__createBinding(result, mod2, k);
|
|
268127
|
+
}
|
|
268128
|
+
__setModuleDefault(result, mod2);
|
|
268129
|
+
return result;
|
|
268130
|
+
};
|
|
268131
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268132
|
+
exports.Address4 = undefined;
|
|
268133
|
+
var common = __importStar(require_common8());
|
|
268134
|
+
var constants = __importStar(require_constants25());
|
|
268135
|
+
var address_error_1 = require_address_error();
|
|
268136
|
+
var isCorrect4 = common.isCorrect(constants.BITS);
|
|
268137
|
+
|
|
268138
|
+
class Address4 {
|
|
268139
|
+
constructor(address) {
|
|
268140
|
+
this.addressMinusSuffix = "";
|
|
268141
|
+
this.groups = constants.GROUPS;
|
|
268142
|
+
this.parsedAddress = [];
|
|
268143
|
+
this.parsedSubnet = "";
|
|
268144
|
+
this.subnet = "/32";
|
|
268145
|
+
this.subnetMask = 32;
|
|
268146
|
+
this.v4 = true;
|
|
268147
|
+
this.isCorrect = isCorrect4;
|
|
268148
|
+
this.isInSubnet = common.isInSubnet;
|
|
268149
|
+
this.isHostInSubnet = common.isHostInSubnet;
|
|
268150
|
+
this.address = address;
|
|
268151
|
+
const subnet = constants.RE_SUBNET_STRING.exec(address);
|
|
268152
|
+
if (subnet) {
|
|
268153
|
+
this.parsedSubnet = subnet[0].replace("/", "");
|
|
268154
|
+
this.subnetMask = parseInt(this.parsedSubnet, 10);
|
|
268155
|
+
this.subnet = `/${this.subnetMask}`;
|
|
268156
|
+
if (this.subnetMask < 0 || this.subnetMask > constants.BITS) {
|
|
268157
|
+
throw new address_error_1.AddressError("Invalid subnet mask.");
|
|
268158
|
+
}
|
|
268159
|
+
address = address.replace(constants.RE_SUBNET_STRING, "");
|
|
268160
|
+
}
|
|
268161
|
+
this.addressMinusSuffix = address;
|
|
268162
|
+
this.parsedAddress = this.parse(address);
|
|
268163
|
+
}
|
|
268164
|
+
static isValid(address) {
|
|
268165
|
+
try {
|
|
268166
|
+
new Address4(address);
|
|
268167
|
+
return true;
|
|
268168
|
+
} catch {
|
|
268169
|
+
return false;
|
|
268170
|
+
}
|
|
268171
|
+
}
|
|
268172
|
+
parse(address) {
|
|
268173
|
+
const groups = address.split(".");
|
|
268174
|
+
if (groups.some((group) => /^0\d/.test(group))) {
|
|
268175
|
+
throw new address_error_1.AddressError("IPv4 addresses can't have leading zeroes.");
|
|
268176
|
+
}
|
|
268177
|
+
if (!address.match(constants.RE_ADDRESS)) {
|
|
268178
|
+
throw new address_error_1.AddressError("Invalid IPv4 address.");
|
|
268179
|
+
}
|
|
268180
|
+
return groups;
|
|
268181
|
+
}
|
|
268182
|
+
correctForm() {
|
|
268183
|
+
return this.parsedAddress.map((part) => parseInt(part, 10)).join(".");
|
|
268184
|
+
}
|
|
268185
|
+
static fromAddressAndMask(address, mask) {
|
|
268186
|
+
const bits = common.prefixLengthFromMask(new Address4(mask).bigInt(), constants.BITS);
|
|
268187
|
+
return new Address4(`${address}/${bits}`);
|
|
268188
|
+
}
|
|
268189
|
+
static fromAddressAndWildcardMask(address, wildcardMask) {
|
|
268190
|
+
const wildcard = new Address4(wildcardMask).bigInt();
|
|
268191
|
+
const allOnes = (BigInt(1) << BigInt(constants.BITS)) - BigInt(1);
|
|
268192
|
+
const mask = wildcard ^ allOnes;
|
|
268193
|
+
const bits = common.prefixLengthFromMask(mask, constants.BITS);
|
|
268194
|
+
return new Address4(`${address}/${bits}`);
|
|
268195
|
+
}
|
|
268196
|
+
static fromWildcard(input) {
|
|
268197
|
+
const groups = input.split(".");
|
|
268198
|
+
if (groups.length !== constants.GROUPS) {
|
|
268199
|
+
throw new address_error_1.AddressError("Wildcard pattern must have 4 octets");
|
|
268200
|
+
}
|
|
268201
|
+
let firstWildcard = -1;
|
|
268202
|
+
for (let i = 0;i < groups.length; i++) {
|
|
268203
|
+
if (groups[i] === "*") {
|
|
268204
|
+
if (firstWildcard === -1) {
|
|
268205
|
+
firstWildcard = i;
|
|
268206
|
+
}
|
|
268207
|
+
} else if (firstWildcard !== -1) {
|
|
268208
|
+
throw new address_error_1.AddressError("Wildcard `*` must only appear in trailing octets (e.g. `192.168.0.*`)");
|
|
268209
|
+
}
|
|
268210
|
+
}
|
|
268211
|
+
const trailing = firstWildcard === -1 ? 0 : groups.length - firstWildcard;
|
|
268212
|
+
const replaced = groups.map((g) => g === "*" ? "0" : g);
|
|
268213
|
+
const subnetBits = constants.BITS - trailing * 8;
|
|
268214
|
+
return new Address4(`${replaced.join(".")}/${subnetBits}`);
|
|
268215
|
+
}
|
|
268216
|
+
static fromHex(hex) {
|
|
268217
|
+
const stripped = hex.replace(/:/g, "");
|
|
268218
|
+
if (!/^[0-9a-fA-F]{8}$/.test(stripped)) {
|
|
268219
|
+
throw new address_error_1.AddressError("IPv4 hex must be exactly 8 hex digits");
|
|
268220
|
+
}
|
|
268221
|
+
const groups = [];
|
|
268222
|
+
for (let i = 0;i < 8; i += 2) {
|
|
268223
|
+
groups.push(parseInt(stripped.slice(i, i + 2), 16));
|
|
268224
|
+
}
|
|
268225
|
+
return new Address4(groups.join("."));
|
|
268226
|
+
}
|
|
268227
|
+
static fromInteger(integer) {
|
|
268228
|
+
if (!Number.isInteger(integer) || integer < 0 || integer > 4294967295) {
|
|
268229
|
+
throw new address_error_1.AddressError("IPv4 integer must be in the range 0 to 2**32 - 1");
|
|
268230
|
+
}
|
|
268231
|
+
return Address4.fromHex(integer.toString(16).padStart(8, "0"));
|
|
268232
|
+
}
|
|
268233
|
+
static fromArpa(arpaFormAddress) {
|
|
268234
|
+
const leader = arpaFormAddress.replace(/(\.in-addr\.arpa)?\.$/, "");
|
|
268235
|
+
const address = leader.split(".").reverse().join(".");
|
|
268236
|
+
return new Address4(address);
|
|
268237
|
+
}
|
|
268238
|
+
toHex() {
|
|
268239
|
+
return this.parsedAddress.map((part) => common.stringToPaddedHex(part)).join(":");
|
|
268240
|
+
}
|
|
268241
|
+
toArray() {
|
|
268242
|
+
return this.parsedAddress.map((part) => parseInt(part, 10));
|
|
268243
|
+
}
|
|
268244
|
+
toGroup6() {
|
|
268245
|
+
const output = [];
|
|
268246
|
+
let i;
|
|
268247
|
+
for (i = 0;i < constants.GROUPS; i += 2) {
|
|
268248
|
+
output.push(`${common.stringToPaddedHex(this.parsedAddress[i])}${common.stringToPaddedHex(this.parsedAddress[i + 1])}`);
|
|
268249
|
+
}
|
|
268250
|
+
return output.join(":");
|
|
268251
|
+
}
|
|
268252
|
+
bigInt() {
|
|
268253
|
+
return BigInt(`0x${this.parsedAddress.map((n) => common.stringToPaddedHex(n)).join("")}`);
|
|
268254
|
+
}
|
|
268255
|
+
_startAddress() {
|
|
268256
|
+
return BigInt(`0b${this.mask() + "0".repeat(constants.BITS - this.subnetMask)}`);
|
|
268257
|
+
}
|
|
268258
|
+
startAddress() {
|
|
268259
|
+
return Address4.fromBigInt(this._startAddress());
|
|
268260
|
+
}
|
|
268261
|
+
startAddressExclusive() {
|
|
268262
|
+
const adjust = BigInt("1");
|
|
268263
|
+
return Address4.fromBigInt(this._startAddress() + adjust);
|
|
268264
|
+
}
|
|
268265
|
+
_endAddress() {
|
|
268266
|
+
return BigInt(`0b${this.mask() + "1".repeat(constants.BITS - this.subnetMask)}`);
|
|
268267
|
+
}
|
|
268268
|
+
endAddress() {
|
|
268269
|
+
return Address4.fromBigInt(this._endAddress());
|
|
268270
|
+
}
|
|
268271
|
+
endAddressExclusive() {
|
|
268272
|
+
const adjust = BigInt("1");
|
|
268273
|
+
return Address4.fromBigInt(this._endAddress() - adjust);
|
|
268274
|
+
}
|
|
268275
|
+
subnetMaskAddress() {
|
|
268276
|
+
return Address4.fromBigInt(BigInt(`0b${"1".repeat(this.subnetMask)}${"0".repeat(constants.BITS - this.subnetMask)}`));
|
|
268277
|
+
}
|
|
268278
|
+
wildcardMask() {
|
|
268279
|
+
return Address4.fromBigInt(BigInt(`0b${"0".repeat(this.subnetMask)}${"1".repeat(constants.BITS - this.subnetMask)}`));
|
|
268280
|
+
}
|
|
268281
|
+
networkForm() {
|
|
268282
|
+
return `${this.startAddress().correctForm()}/${this.subnetMask}`;
|
|
268283
|
+
}
|
|
268284
|
+
static fromBigInt(bigInt) {
|
|
268285
|
+
if (bigInt < BigInt(0) || bigInt > BigInt(4294967295)) {
|
|
268286
|
+
throw new address_error_1.AddressError("IPv4 BigInt must be in the range 0 to 2**32 - 1");
|
|
268287
|
+
}
|
|
268288
|
+
return Address4.fromHex(bigInt.toString(16).padStart(8, "0"));
|
|
268289
|
+
}
|
|
268290
|
+
static fromByteArray(bytes) {
|
|
268291
|
+
common.assertByteArray(bytes, 4, "IPv4", 0);
|
|
268292
|
+
return this.fromUnsignedByteArray(bytes);
|
|
268293
|
+
}
|
|
268294
|
+
static fromUnsignedByteArray(bytes) {
|
|
268295
|
+
if (bytes.length !== 4) {
|
|
268296
|
+
throw new address_error_1.AddressError("IPv4 addresses require exactly 4 bytes");
|
|
268297
|
+
}
|
|
268298
|
+
const address = bytes.join(".");
|
|
268299
|
+
return new Address4(address);
|
|
268300
|
+
}
|
|
268301
|
+
mask(mask) {
|
|
268302
|
+
if (mask === undefined) {
|
|
268303
|
+
mask = this.subnetMask;
|
|
268304
|
+
}
|
|
268305
|
+
return this.getBitsBase2(0, mask);
|
|
268306
|
+
}
|
|
268307
|
+
getBitsBase2(start, end) {
|
|
268308
|
+
return this.binaryZeroPad().slice(start, end);
|
|
268309
|
+
}
|
|
268310
|
+
reverseForm(options) {
|
|
268311
|
+
if (!options) {
|
|
268312
|
+
options = {};
|
|
268313
|
+
}
|
|
268314
|
+
const reversed = this.correctForm().split(".").reverse().join(".");
|
|
268315
|
+
if (options.omitSuffix) {
|
|
268316
|
+
return reversed;
|
|
268317
|
+
}
|
|
268318
|
+
return `${reversed}.in-addr.arpa.`;
|
|
268319
|
+
}
|
|
268320
|
+
isMulticast() {
|
|
268321
|
+
return this.isHostInSubnet(MULTICAST_V4);
|
|
268322
|
+
}
|
|
268323
|
+
isPrivate() {
|
|
268324
|
+
return PRIVATE_V4.some((subnet) => this.isHostInSubnet(subnet));
|
|
268325
|
+
}
|
|
268326
|
+
isLoopback() {
|
|
268327
|
+
return this.isHostInSubnet(LOOPBACK_V4);
|
|
268328
|
+
}
|
|
268329
|
+
isLinkLocal() {
|
|
268330
|
+
return this.isHostInSubnet(LINK_LOCAL_V4);
|
|
268331
|
+
}
|
|
268332
|
+
isUnspecified() {
|
|
268333
|
+
return this.isHostInSubnet(UNSPECIFIED_V4);
|
|
268334
|
+
}
|
|
268335
|
+
isBroadcast() {
|
|
268336
|
+
return this.isHostInSubnet(BROADCAST_V4);
|
|
268337
|
+
}
|
|
268338
|
+
isCGNAT() {
|
|
268339
|
+
return this.isHostInSubnet(CGNAT_V4);
|
|
268340
|
+
}
|
|
268341
|
+
binaryZeroPad() {
|
|
268342
|
+
if (this._binaryZeroPad === undefined) {
|
|
268343
|
+
this._binaryZeroPad = this.bigInt().toString(2).padStart(constants.BITS, "0");
|
|
268344
|
+
}
|
|
268345
|
+
return this._binaryZeroPad;
|
|
268346
|
+
}
|
|
268347
|
+
groupForV6() {
|
|
268348
|
+
const segments = this.parsedAddress;
|
|
268349
|
+
return this.correctForm().replace(constants.RE_ADDRESS, `<span class="hover-group group-v4 group-6">${segments.slice(0, 2).join(".")}</span>.<span class="hover-group group-v4 group-7">${segments.slice(2, 4).join(".")}</span>`);
|
|
268350
|
+
}
|
|
268351
|
+
}
|
|
268352
|
+
exports.Address4 = Address4;
|
|
268353
|
+
var MULTICAST_V4 = new Address4("224.0.0.0/4");
|
|
268354
|
+
var PRIVATE_V4 = [
|
|
268355
|
+
new Address4("10.0.0.0/8"),
|
|
268356
|
+
new Address4("172.16.0.0/12"),
|
|
268357
|
+
new Address4("192.168.0.0/16")
|
|
268358
|
+
];
|
|
268359
|
+
var LOOPBACK_V4 = new Address4("127.0.0.0/8");
|
|
268360
|
+
var LINK_LOCAL_V4 = new Address4("169.254.0.0/16");
|
|
268361
|
+
var UNSPECIFIED_V4 = new Address4("0.0.0.0/32");
|
|
268362
|
+
var BROADCAST_V4 = new Address4("255.255.255.255/32");
|
|
268363
|
+
var CGNAT_V4 = new Address4("100.64.0.0/10");
|
|
268364
|
+
});
|
|
268365
|
+
|
|
268366
|
+
// ../../node_modules/ip-address/dist/v6/constants.js
|
|
268367
|
+
var require_constants26 = __commonJS((exports) => {
|
|
268368
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268369
|
+
exports.RE_URL_WITH_PORT = exports.RE_URL = exports.RE_ZONE_STRING = exports.RE_SUBNET_STRING = exports.RE_BAD_ADDRESS = exports.RE_BAD_CHARACTERS = exports.TYPES = exports.SCOPES = exports.GROUPS = exports.BITS = undefined;
|
|
268370
|
+
exports.BITS = 128;
|
|
268371
|
+
exports.GROUPS = 8;
|
|
268372
|
+
exports.SCOPES = {
|
|
268373
|
+
0: "Reserved",
|
|
268374
|
+
1: "Interface local",
|
|
268375
|
+
2: "Link local",
|
|
268376
|
+
4: "Admin local",
|
|
268377
|
+
5: "Site local",
|
|
268378
|
+
8: "Organization local",
|
|
268379
|
+
14: "Global",
|
|
268380
|
+
15: "Reserved"
|
|
268381
|
+
};
|
|
268382
|
+
exports.TYPES = {
|
|
268383
|
+
"ff01::1/128": "Multicast (All nodes on this interface)",
|
|
268384
|
+
"ff01::2/128": "Multicast (All routers on this interface)",
|
|
268385
|
+
"ff02::1/128": "Multicast (All nodes on this link)",
|
|
268386
|
+
"ff02::2/128": "Multicast (All routers on this link)",
|
|
268387
|
+
"ff05::2/128": "Multicast (All routers in this site)",
|
|
268388
|
+
"ff02::5/128": "Multicast (OSPFv3 AllSPF routers)",
|
|
268389
|
+
"ff02::6/128": "Multicast (OSPFv3 AllDR routers)",
|
|
268390
|
+
"ff02::9/128": "Multicast (RIP routers)",
|
|
268391
|
+
"ff02::a/128": "Multicast (EIGRP routers)",
|
|
268392
|
+
"ff02::d/128": "Multicast (PIM routers)",
|
|
268393
|
+
"ff02::16/128": "Multicast (MLDv2 reports)",
|
|
268394
|
+
"ff01::fb/128": "Multicast (mDNSv6)",
|
|
268395
|
+
"ff02::fb/128": "Multicast (mDNSv6)",
|
|
268396
|
+
"ff05::fb/128": "Multicast (mDNSv6)",
|
|
268397
|
+
"ff02::1:2/128": "Multicast (All DHCP servers and relay agents on this link)",
|
|
268398
|
+
"ff05::1:2/128": "Multicast (All DHCP servers and relay agents in this site)",
|
|
268399
|
+
"ff02::1:3/128": "Multicast (All DHCP servers on this link)",
|
|
268400
|
+
"ff05::1:3/128": "Multicast (All DHCP servers in this site)",
|
|
268401
|
+
"::/128": "Unspecified",
|
|
268402
|
+
"::1/128": "Loopback",
|
|
268403
|
+
"::ffff:0:0/96": "IPv4-mapped",
|
|
268404
|
+
"ff00::/8": "Multicast",
|
|
268405
|
+
"fe80::/10": "Link-local unicast",
|
|
268406
|
+
"fc00::/7": "Unique local",
|
|
268407
|
+
"2002::/16": "6to4",
|
|
268408
|
+
"2001:db8::/32": "Documentation",
|
|
268409
|
+
"64:ff9b::/96": "NAT64 (well-known)",
|
|
268410
|
+
"64:ff9b:1::/48": "NAT64 (local-use)"
|
|
268411
|
+
};
|
|
268412
|
+
exports.RE_BAD_CHARACTERS = /([^0-9a-f:/%])/gi;
|
|
268413
|
+
exports.RE_BAD_ADDRESS = /([0-9a-f]{5,}|:{3,}|[^:]:$|^:[^:]|\/$)/gi;
|
|
268414
|
+
exports.RE_SUBNET_STRING = /\/\d{1,3}(?=%|$)/;
|
|
268415
|
+
exports.RE_ZONE_STRING = /%.*$/;
|
|
268416
|
+
exports.RE_URL = /^(?:\[([0-9a-f:.]+)\]|([0-9a-f:.]+))(?:[/?#].*)?$/i;
|
|
268417
|
+
exports.RE_URL_WITH_PORT = /^\[([0-9a-f:.]+)\]:([0-9]{1,5})(?:[/?#].*)?$/i;
|
|
268418
|
+
});
|
|
268419
|
+
|
|
268420
|
+
// ../../node_modules/ip-address/dist/v6/helpers.js
|
|
268421
|
+
var require_helpers2 = __commonJS((exports) => {
|
|
268422
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268423
|
+
exports.escapeHtml = escapeHtml;
|
|
268424
|
+
exports.spanAllZeroes = spanAllZeroes;
|
|
268425
|
+
exports.spanAll = spanAll;
|
|
268426
|
+
exports.spanLeadingZeroes = spanLeadingZeroes;
|
|
268427
|
+
exports.simpleGroup = simpleGroup;
|
|
268428
|
+
function escapeHtml(s) {
|
|
268429
|
+
return s.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
268430
|
+
}
|
|
268431
|
+
function spanAllZeroes(s) {
|
|
268432
|
+
return escapeHtml(s).replace(/(0+)/g, '<span class="zero">$1</span>');
|
|
268433
|
+
}
|
|
268434
|
+
function spanAll(s, offset = 0) {
|
|
268435
|
+
const letters = s.split("");
|
|
268436
|
+
return letters.map((n, i) => `<span class="digit value-${escapeHtml(n)} position-${i + offset}">${spanAllZeroes(n)}</span>`).join("");
|
|
268437
|
+
}
|
|
268438
|
+
function spanLeadingZeroesSimple(group) {
|
|
268439
|
+
return escapeHtml(group).replace(/^(0+)/, '<span class="zero">$1</span>');
|
|
268440
|
+
}
|
|
268441
|
+
function spanLeadingZeroes(address) {
|
|
268442
|
+
const groups = address.split(":");
|
|
268443
|
+
return groups.map((g) => spanLeadingZeroesSimple(g)).join(":");
|
|
268444
|
+
}
|
|
268445
|
+
function simpleGroup(addressString, offset = 0) {
|
|
268446
|
+
const groups = addressString.split(":");
|
|
268447
|
+
return groups.map((g, i) => {
|
|
268448
|
+
if (/group-v4/.test(g)) {
|
|
268449
|
+
return g;
|
|
268450
|
+
}
|
|
268451
|
+
return `<span class="hover-group group-${i + offset}">${spanLeadingZeroesSimple(g)}</span>`;
|
|
268452
|
+
});
|
|
268453
|
+
}
|
|
268454
|
+
});
|
|
268455
|
+
|
|
268456
|
+
// ../../node_modules/ip-address/dist/v6/regular-expressions.js
|
|
268457
|
+
var require_regular_expressions = __commonJS((exports) => {
|
|
268458
|
+
var __createBinding = exports && exports.__createBinding || (Object.create ? function(o, m, k, k2) {
|
|
268459
|
+
if (k2 === undefined)
|
|
268460
|
+
k2 = k;
|
|
268461
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
268462
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
268463
|
+
desc = { enumerable: true, get: function() {
|
|
268464
|
+
return m[k];
|
|
268465
|
+
} };
|
|
268466
|
+
}
|
|
268467
|
+
Object.defineProperty(o, k2, desc);
|
|
268468
|
+
} : function(o, m, k, k2) {
|
|
268469
|
+
if (k2 === undefined)
|
|
268470
|
+
k2 = k;
|
|
268471
|
+
o[k2] = m[k];
|
|
268472
|
+
});
|
|
268473
|
+
var __setModuleDefault = exports && exports.__setModuleDefault || (Object.create ? function(o, v) {
|
|
268474
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
268475
|
+
} : function(o, v) {
|
|
268476
|
+
o["default"] = v;
|
|
268477
|
+
});
|
|
268478
|
+
var __importStar = exports && exports.__importStar || function(mod2) {
|
|
268479
|
+
if (mod2 && mod2.__esModule)
|
|
268480
|
+
return mod2;
|
|
268481
|
+
var result = {};
|
|
268482
|
+
if (mod2 != null) {
|
|
268483
|
+
for (var k in mod2)
|
|
268484
|
+
if (k !== "default" && Object.prototype.hasOwnProperty.call(mod2, k))
|
|
268485
|
+
__createBinding(result, mod2, k);
|
|
268486
|
+
}
|
|
268487
|
+
__setModuleDefault(result, mod2);
|
|
268488
|
+
return result;
|
|
268489
|
+
};
|
|
268490
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268491
|
+
exports.ADDRESS_BOUNDARY = undefined;
|
|
268492
|
+
exports.groupPossibilities = groupPossibilities;
|
|
268493
|
+
exports.padGroup = padGroup;
|
|
268494
|
+
exports.simpleRegularExpression = simpleRegularExpression;
|
|
268495
|
+
exports.possibleElisions = possibleElisions;
|
|
268496
|
+
var v6 = __importStar(require_constants26());
|
|
268497
|
+
function groupPossibilities(possibilities) {
|
|
268498
|
+
return `(${possibilities.join("|")})`;
|
|
268499
|
+
}
|
|
268500
|
+
function padGroup(group) {
|
|
268501
|
+
if (group.length < 4) {
|
|
268502
|
+
return `0{0,${4 - group.length}}${group}`;
|
|
268503
|
+
}
|
|
268504
|
+
return group;
|
|
268505
|
+
}
|
|
268506
|
+
exports.ADDRESS_BOUNDARY = "[^A-Fa-f0-9:]";
|
|
268507
|
+
function simpleRegularExpression(groups) {
|
|
268508
|
+
const zeroIndexes = [];
|
|
268509
|
+
groups.forEach((group, i) => {
|
|
268510
|
+
const groupInteger = parseInt(group, 16);
|
|
268511
|
+
if (groupInteger === 0) {
|
|
268512
|
+
zeroIndexes.push(i);
|
|
268513
|
+
}
|
|
268514
|
+
});
|
|
268515
|
+
const possibilities = zeroIndexes.map((zeroIndex) => groups.map((group, i) => {
|
|
268516
|
+
if (i === zeroIndex) {
|
|
268517
|
+
const elision = i === 0 || i === v6.GROUPS - 1 ? ":" : "";
|
|
268518
|
+
return groupPossibilities([padGroup(group), elision]);
|
|
268519
|
+
}
|
|
268520
|
+
return padGroup(group);
|
|
268521
|
+
}).join(":"));
|
|
268522
|
+
possibilities.push(groups.map(padGroup).join(":"));
|
|
268523
|
+
return groupPossibilities(possibilities);
|
|
268524
|
+
}
|
|
268525
|
+
function possibleElisions(elidedGroups, moreLeft, moreRight) {
|
|
268526
|
+
const left = moreLeft ? "" : ":";
|
|
268527
|
+
const right = moreRight ? "" : ":";
|
|
268528
|
+
const possibilities = [];
|
|
268529
|
+
if (!moreLeft && !moreRight) {
|
|
268530
|
+
possibilities.push("::");
|
|
268531
|
+
}
|
|
268532
|
+
if (moreLeft && moreRight) {
|
|
268533
|
+
possibilities.push("");
|
|
268534
|
+
}
|
|
268535
|
+
if (moreRight && !moreLeft || !moreRight && moreLeft) {
|
|
268536
|
+
possibilities.push(":");
|
|
268537
|
+
}
|
|
268538
|
+
possibilities.push(`${left}(:0{1,4}){1,${elidedGroups - 1}}`);
|
|
268539
|
+
possibilities.push(`(0{1,4}:){1,${elidedGroups - 1}}${right}`);
|
|
268540
|
+
possibilities.push(`(0{1,4}:){${elidedGroups - 1}}0{1,4}`);
|
|
268541
|
+
for (let groups = 1;groups < elidedGroups - 1; groups++) {
|
|
268542
|
+
for (let position = 1;position < elidedGroups - groups; position++) {
|
|
268543
|
+
possibilities.push(`(0{1,4}:){${position}}:(0{1,4}:){${elidedGroups - position - groups - 1}}0{1,4}`);
|
|
268544
|
+
}
|
|
268545
|
+
}
|
|
268546
|
+
return groupPossibilities(possibilities);
|
|
268547
|
+
}
|
|
268548
|
+
});
|
|
268549
|
+
|
|
268550
|
+
// ../../node_modules/ip-address/dist/ipv6.js
|
|
268551
|
+
var require_ipv6 = __commonJS((exports) => {
|
|
268552
|
+
var __createBinding = exports && exports.__createBinding || (Object.create ? function(o, m, k, k2) {
|
|
268553
|
+
if (k2 === undefined)
|
|
268554
|
+
k2 = k;
|
|
268555
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
268556
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
268557
|
+
desc = { enumerable: true, get: function() {
|
|
268558
|
+
return m[k];
|
|
268559
|
+
} };
|
|
268560
|
+
}
|
|
268561
|
+
Object.defineProperty(o, k2, desc);
|
|
268562
|
+
} : function(o, m, k, k2) {
|
|
268563
|
+
if (k2 === undefined)
|
|
268564
|
+
k2 = k;
|
|
268565
|
+
o[k2] = m[k];
|
|
268566
|
+
});
|
|
268567
|
+
var __setModuleDefault = exports && exports.__setModuleDefault || (Object.create ? function(o, v) {
|
|
268568
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
268569
|
+
} : function(o, v) {
|
|
268570
|
+
o["default"] = v;
|
|
268571
|
+
});
|
|
268572
|
+
var __importStar = exports && exports.__importStar || function(mod2) {
|
|
268573
|
+
if (mod2 && mod2.__esModule)
|
|
268574
|
+
return mod2;
|
|
268575
|
+
var result = {};
|
|
268576
|
+
if (mod2 != null) {
|
|
268577
|
+
for (var k in mod2)
|
|
268578
|
+
if (k !== "default" && Object.prototype.hasOwnProperty.call(mod2, k))
|
|
268579
|
+
__createBinding(result, mod2, k);
|
|
268580
|
+
}
|
|
268581
|
+
__setModuleDefault(result, mod2);
|
|
268582
|
+
return result;
|
|
268583
|
+
};
|
|
268584
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
268585
|
+
exports.Address6 = undefined;
|
|
268586
|
+
var common = __importStar(require_common8());
|
|
268587
|
+
var constants4 = __importStar(require_constants25());
|
|
268588
|
+
var constants6 = __importStar(require_constants26());
|
|
268589
|
+
var helpers = __importStar(require_helpers2());
|
|
268590
|
+
var ipv4_1 = require_ipv4();
|
|
268591
|
+
var regular_expressions_1 = require_regular_expressions();
|
|
268592
|
+
var address_error_1 = require_address_error();
|
|
268593
|
+
var common_1 = require_common8();
|
|
268594
|
+
var isCorrect6 = common.isCorrect(constants6.BITS);
|
|
268595
|
+
function assert(condition) {
|
|
268596
|
+
if (!condition) {
|
|
268597
|
+
throw new Error("Assertion failed.");
|
|
268598
|
+
}
|
|
268599
|
+
}
|
|
268600
|
+
function addCommas(number2) {
|
|
268601
|
+
const r = /(\d+)(\d{3})/;
|
|
268602
|
+
while (r.test(number2)) {
|
|
268603
|
+
number2 = number2.replace(r, "$1,$2");
|
|
268604
|
+
}
|
|
268605
|
+
return number2;
|
|
268606
|
+
}
|
|
268607
|
+
function spanLeadingZeroes4(n) {
|
|
268608
|
+
n = n.replace(/^(0{1,})([1-9]+)$/, '<span class="parse-error">$1</span>$2');
|
|
268609
|
+
n = n.replace(/^(0{1,})(0)$/, '<span class="parse-error">$1</span>$2');
|
|
268610
|
+
return n;
|
|
268611
|
+
}
|
|
268612
|
+
function compact(address, slice) {
|
|
268613
|
+
const s1 = [];
|
|
268614
|
+
const s2 = [];
|
|
268615
|
+
let i;
|
|
268616
|
+
for (i = 0;i < address.length; i++) {
|
|
268617
|
+
if (i < slice[0]) {
|
|
268618
|
+
s1.push(address[i]);
|
|
268619
|
+
} else if (i > slice[1]) {
|
|
268620
|
+
s2.push(address[i]);
|
|
268621
|
+
}
|
|
268622
|
+
}
|
|
268623
|
+
return s1.concat(["compact"]).concat(s2);
|
|
268624
|
+
}
|
|
268625
|
+
function paddedHex(octet) {
|
|
268626
|
+
return parseInt(octet, 16).toString(16).padStart(4, "0");
|
|
268627
|
+
}
|
|
268628
|
+
function unsignByte(b) {
|
|
268629
|
+
return b & 255;
|
|
268630
|
+
}
|
|
268631
|
+
|
|
268632
|
+
class Address6 {
|
|
268633
|
+
constructor(address, optionalGroups) {
|
|
268634
|
+
this.addressMinusSuffix = "";
|
|
268635
|
+
this.parsedSubnet = "";
|
|
268636
|
+
this.subnet = "/128";
|
|
268637
|
+
this.subnetMask = 128;
|
|
268638
|
+
this.v4 = false;
|
|
268639
|
+
this.zone = "";
|
|
268640
|
+
this.isInSubnet = common.isInSubnet;
|
|
268641
|
+
this.isHostInSubnet = common.isHostInSubnet;
|
|
268642
|
+
this.isCorrect = isCorrect6;
|
|
268643
|
+
if (optionalGroups === undefined) {
|
|
268644
|
+
this.groups = constants6.GROUPS;
|
|
268645
|
+
} else {
|
|
268646
|
+
this.groups = optionalGroups;
|
|
268647
|
+
}
|
|
268648
|
+
this.address = address;
|
|
268649
|
+
const subnet = constants6.RE_SUBNET_STRING.exec(address);
|
|
268650
|
+
if (subnet) {
|
|
268651
|
+
this.parsedSubnet = subnet[0].replace("/", "");
|
|
268652
|
+
this.subnetMask = parseInt(this.parsedSubnet, 10);
|
|
268653
|
+
this.subnet = `/${this.subnetMask}`;
|
|
268654
|
+
if (Number.isNaN(this.subnetMask) || this.subnetMask < 0 || this.subnetMask > constants6.BITS) {
|
|
268655
|
+
throw new address_error_1.AddressError("Invalid subnet mask.");
|
|
268656
|
+
}
|
|
268657
|
+
address = address.replace(constants6.RE_SUBNET_STRING, "");
|
|
268658
|
+
}
|
|
268659
|
+
if (/\//.test(address)) {
|
|
268660
|
+
throw new address_error_1.AddressError("Invalid subnet mask.");
|
|
268661
|
+
}
|
|
268662
|
+
const zone = constants6.RE_ZONE_STRING.exec(address);
|
|
268663
|
+
if (zone) {
|
|
268664
|
+
this.zone = zone[0];
|
|
268665
|
+
address = address.replace(constants6.RE_ZONE_STRING, "");
|
|
268666
|
+
}
|
|
268667
|
+
this.addressMinusSuffix = address;
|
|
268668
|
+
this.parsedAddress = this.parse(this.addressMinusSuffix);
|
|
268669
|
+
}
|
|
268670
|
+
static isValid(address) {
|
|
268671
|
+
try {
|
|
268672
|
+
new Address6(address);
|
|
268673
|
+
return true;
|
|
268674
|
+
} catch {
|
|
268675
|
+
return false;
|
|
268676
|
+
}
|
|
268677
|
+
}
|
|
268678
|
+
static fromBigInt(bigInt) {
|
|
268679
|
+
if (bigInt < BigInt(0) || bigInt > (BigInt(1) << BigInt(constants6.BITS)) - BigInt(1)) {
|
|
268680
|
+
throw new address_error_1.AddressError("IPv6 BigInt must be in the range 0 to 2**128 - 1");
|
|
268681
|
+
}
|
|
268682
|
+
const hex = bigInt.toString(16).padStart(32, "0");
|
|
268683
|
+
const groups = [];
|
|
268684
|
+
for (let i = 0;i < constants6.GROUPS; i++) {
|
|
268685
|
+
groups.push(hex.slice(i * 4, (i + 1) * 4));
|
|
268686
|
+
}
|
|
268687
|
+
return new Address6(groups.join(":"));
|
|
268688
|
+
}
|
|
268689
|
+
static fromURL(url2) {
|
|
268690
|
+
var _a2;
|
|
268691
|
+
let host;
|
|
268692
|
+
let port = null;
|
|
268693
|
+
let result;
|
|
268694
|
+
let error;
|
|
268695
|
+
const stripped = url2.replace(/^[a-z][a-z0-9+.-]*:\/\//i, "");
|
|
268696
|
+
if (stripped.indexOf("[") !== -1 && stripped.indexOf("]:") !== -1) {
|
|
268697
|
+
error = "failed to parse address with port";
|
|
268698
|
+
result = constants6.RE_URL_WITH_PORT.exec(stripped);
|
|
268699
|
+
if (result === null) {
|
|
268700
|
+
return { error, address: null, port: null };
|
|
268701
|
+
}
|
|
268702
|
+
host = result[1];
|
|
268703
|
+
port = result[2];
|
|
268704
|
+
} else {
|
|
268705
|
+
error = "failed to parse address from URL";
|
|
268706
|
+
result = constants6.RE_URL.exec(stripped);
|
|
268707
|
+
if (result === null) {
|
|
268708
|
+
return { error, address: null, port: null };
|
|
268709
|
+
}
|
|
268710
|
+
host = (_a2 = result[1]) !== null && _a2 !== undefined ? _a2 : result[2];
|
|
268711
|
+
}
|
|
268712
|
+
if (port) {
|
|
268713
|
+
port = parseInt(port, 10);
|
|
268714
|
+
if (port < 0 || port > 65535) {
|
|
268715
|
+
port = null;
|
|
268716
|
+
}
|
|
268717
|
+
} else {
|
|
268718
|
+
port = null;
|
|
268719
|
+
}
|
|
268720
|
+
let address;
|
|
268721
|
+
try {
|
|
268722
|
+
address = new Address6(host);
|
|
268723
|
+
} catch {
|
|
268724
|
+
return { error, address: null, port: null };
|
|
268725
|
+
}
|
|
268726
|
+
return { address, port };
|
|
268727
|
+
}
|
|
268728
|
+
static fromAddressAndMask(address, mask) {
|
|
268729
|
+
const bits = common.prefixLengthFromMask(new Address6(mask).bigInt(), constants6.BITS);
|
|
268730
|
+
return new Address6(`${address}/${bits}`);
|
|
268731
|
+
}
|
|
268732
|
+
static fromAddressAndWildcardMask(address, wildcardMask) {
|
|
268733
|
+
const wildcard = new Address6(wildcardMask).bigInt();
|
|
268734
|
+
const allOnes = (BigInt(1) << BigInt(constants6.BITS)) - BigInt(1);
|
|
268735
|
+
const mask = wildcard ^ allOnes;
|
|
268736
|
+
const bits = common.prefixLengthFromMask(mask, constants6.BITS);
|
|
268737
|
+
return new Address6(`${address}/${bits}`);
|
|
268738
|
+
}
|
|
268739
|
+
static fromWildcard(input) {
|
|
268740
|
+
if (input.includes("%") || input.includes("/")) {
|
|
268741
|
+
throw new address_error_1.AddressError("Wildcard pattern must not include a zone or CIDR suffix");
|
|
268742
|
+
}
|
|
268743
|
+
const halves = input.split("::");
|
|
268744
|
+
if (halves.length > 2) {
|
|
268745
|
+
throw new address_error_1.AddressError("Wildcard pattern cannot contain more than one '::'");
|
|
268746
|
+
}
|
|
268747
|
+
let groups;
|
|
268748
|
+
if (halves.length === 2) {
|
|
268749
|
+
const left = halves[0] === "" ? [] : halves[0].split(":");
|
|
268750
|
+
const right = halves[1] === "" ? [] : halves[1].split(":");
|
|
268751
|
+
const remaining = constants6.GROUPS - left.length - right.length;
|
|
268752
|
+
if (remaining < 1) {
|
|
268753
|
+
throw new address_error_1.AddressError("Wildcard pattern with '::' has too many groups");
|
|
268754
|
+
}
|
|
268755
|
+
groups = [...left, ...new Array(remaining).fill("0"), ...right];
|
|
268756
|
+
} else {
|
|
268757
|
+
groups = input.split(":");
|
|
268758
|
+
}
|
|
268759
|
+
if (groups.length !== constants6.GROUPS) {
|
|
268760
|
+
throw new address_error_1.AddressError("Wildcard pattern must have 8 groups");
|
|
268761
|
+
}
|
|
268762
|
+
let firstWildcard = -1;
|
|
268763
|
+
for (let i = 0;i < groups.length; i++) {
|
|
268764
|
+
if (groups[i] === "*") {
|
|
268765
|
+
if (firstWildcard === -1) {
|
|
268766
|
+
firstWildcard = i;
|
|
268767
|
+
}
|
|
268768
|
+
} else if (firstWildcard !== -1) {
|
|
268769
|
+
throw new address_error_1.AddressError("Wildcard `*` must only appear in trailing groups (e.g. `2001:db8:*:*:*:*:*:*`)");
|
|
268770
|
+
}
|
|
268771
|
+
}
|
|
268772
|
+
const trailing = firstWildcard === -1 ? 0 : groups.length - firstWildcard;
|
|
268773
|
+
const replaced = groups.map((g) => g === "*" ? "0" : g);
|
|
268774
|
+
const subnetBits = constants6.BITS - trailing * 16;
|
|
268775
|
+
return new Address6(`${replaced.join(":")}/${subnetBits}`);
|
|
268776
|
+
}
|
|
268777
|
+
static fromAddress4(address) {
|
|
268778
|
+
const address4 = new ipv4_1.Address4(address);
|
|
268779
|
+
const mask6 = constants6.BITS - (constants4.BITS - address4.subnetMask);
|
|
268780
|
+
return new Address6(`::ffff:${address4.correctForm()}/${mask6}`);
|
|
268781
|
+
}
|
|
268782
|
+
static fromArpa(arpaFormAddress) {
|
|
268783
|
+
let address = arpaFormAddress.replace(/(\.ip6\.arpa)?\.$/, "");
|
|
268784
|
+
const semicolonAmount = 7;
|
|
268785
|
+
if (address.length !== 63) {
|
|
268786
|
+
throw new address_error_1.AddressError("Invalid 'ip6.arpa' form.");
|
|
268787
|
+
}
|
|
268788
|
+
const parts = address.split(".").reverse();
|
|
268789
|
+
for (let i = semicolonAmount;i > 0; i--) {
|
|
268790
|
+
const insertIndex = i * 4;
|
|
268791
|
+
parts.splice(insertIndex, 0, ":");
|
|
268792
|
+
}
|
|
268793
|
+
address = parts.join("");
|
|
268794
|
+
return new Address6(address);
|
|
268795
|
+
}
|
|
268796
|
+
microsoftTranscription() {
|
|
268797
|
+
return `${this.correctForm().replace(/:/g, "-")}.ipv6-literal.net`;
|
|
268798
|
+
}
|
|
268799
|
+
mask(mask = this.subnetMask) {
|
|
268800
|
+
return this.getBitsBase2(0, mask);
|
|
268801
|
+
}
|
|
268802
|
+
possibleSubnets(subnetSize = 128) {
|
|
268803
|
+
const availableBits = constants6.BITS - this.subnetMask;
|
|
268804
|
+
const subnetBits = Math.abs(subnetSize - constants6.BITS);
|
|
268805
|
+
const subnetPowers = availableBits - subnetBits;
|
|
268806
|
+
if (subnetPowers < 0) {
|
|
268807
|
+
return "0";
|
|
268808
|
+
}
|
|
268809
|
+
return addCommas((BigInt("2") ** BigInt(subnetPowers)).toString(10));
|
|
268810
|
+
}
|
|
268811
|
+
_startAddress() {
|
|
268812
|
+
return BigInt(`0b${this.mask() + "0".repeat(constants6.BITS - this.subnetMask)}`);
|
|
268813
|
+
}
|
|
268814
|
+
startAddress() {
|
|
268815
|
+
return Address6.fromBigInt(this._startAddress());
|
|
268816
|
+
}
|
|
268817
|
+
startAddressExclusive() {
|
|
268818
|
+
const adjust = BigInt("1");
|
|
268819
|
+
return Address6.fromBigInt(this._startAddress() + adjust);
|
|
268820
|
+
}
|
|
268821
|
+
_endAddress() {
|
|
268822
|
+
return BigInt(`0b${this.mask() + "1".repeat(constants6.BITS - this.subnetMask)}`);
|
|
268823
|
+
}
|
|
268824
|
+
endAddress() {
|
|
268825
|
+
return Address6.fromBigInt(this._endAddress());
|
|
268826
|
+
}
|
|
268827
|
+
endAddressExclusive() {
|
|
268828
|
+
const adjust = BigInt("1");
|
|
268829
|
+
return Address6.fromBigInt(this._endAddress() - adjust);
|
|
268830
|
+
}
|
|
268831
|
+
subnetMaskAddress() {
|
|
268832
|
+
return Address6.fromBigInt(BigInt(`0b${"1".repeat(this.subnetMask)}${"0".repeat(constants6.BITS - this.subnetMask)}`));
|
|
268833
|
+
}
|
|
268834
|
+
wildcardMask() {
|
|
268835
|
+
return Address6.fromBigInt(BigInt(`0b${"0".repeat(this.subnetMask)}${"1".repeat(constants6.BITS - this.subnetMask)}`));
|
|
268836
|
+
}
|
|
268837
|
+
networkForm() {
|
|
268838
|
+
return `${this.startAddress().correctForm()}/${this.subnetMask}`;
|
|
268839
|
+
}
|
|
268840
|
+
getScope() {
|
|
268841
|
+
const type = this.getType();
|
|
268842
|
+
if (type === "Multicast" || type.startsWith("Multicast ")) {
|
|
268843
|
+
const scope = constants6.SCOPES[parseInt(this.getBits(12, 16).toString(10), 10)];
|
|
268844
|
+
return scope || "Unknown";
|
|
268845
|
+
}
|
|
268846
|
+
if (type === "Link-local unicast" || type === "Loopback") {
|
|
268847
|
+
return "Link local";
|
|
268848
|
+
}
|
|
268849
|
+
if (type === "Unspecified") {
|
|
268850
|
+
return "Unknown";
|
|
268851
|
+
}
|
|
268852
|
+
return "Global";
|
|
268853
|
+
}
|
|
268854
|
+
getType() {
|
|
268855
|
+
for (let i = 0;i < TYPE_SUBNETS.length; i++) {
|
|
268856
|
+
const entry = TYPE_SUBNETS[i];
|
|
268857
|
+
if (this.isHostInSubnet(entry[0])) {
|
|
268858
|
+
return entry[1];
|
|
268859
|
+
}
|
|
268860
|
+
}
|
|
268861
|
+
return "Global unicast";
|
|
268862
|
+
}
|
|
268863
|
+
getBits(start, end) {
|
|
268864
|
+
return BigInt(`0b${this.getBitsBase2(start, end)}`);
|
|
268865
|
+
}
|
|
268866
|
+
getBitsBase2(start, end) {
|
|
268867
|
+
return this.binaryZeroPad().slice(start, end);
|
|
268868
|
+
}
|
|
268869
|
+
getBitsBase16(start, end) {
|
|
268870
|
+
const length = end - start;
|
|
268871
|
+
if (length % 4 !== 0) {
|
|
268872
|
+
throw new Error("Length of bits to retrieve must be divisible by four");
|
|
268873
|
+
}
|
|
268874
|
+
return this.getBits(start, end).toString(16).padStart(length / 4, "0");
|
|
268875
|
+
}
|
|
268876
|
+
getBitsPastSubnet() {
|
|
268877
|
+
return this.getBitsBase2(this.subnetMask, constants6.BITS);
|
|
268878
|
+
}
|
|
268879
|
+
reverseForm(options) {
|
|
268880
|
+
if (!options) {
|
|
268881
|
+
options = {};
|
|
268882
|
+
}
|
|
268883
|
+
const characters = Math.floor(this.subnetMask / 4);
|
|
268884
|
+
const reversed = this.canonicalForm().replace(/:/g, "").split("").slice(0, characters).reverse().join(".");
|
|
268885
|
+
if (characters > 0) {
|
|
268886
|
+
if (options.omitSuffix) {
|
|
268887
|
+
return reversed;
|
|
268888
|
+
}
|
|
268889
|
+
return `${reversed}.ip6.arpa.`;
|
|
268890
|
+
}
|
|
268891
|
+
if (options.omitSuffix) {
|
|
268892
|
+
return "";
|
|
268893
|
+
}
|
|
268894
|
+
return "ip6.arpa.";
|
|
268895
|
+
}
|
|
268896
|
+
correctForm() {
|
|
268897
|
+
let i;
|
|
268898
|
+
let groups = [];
|
|
268899
|
+
let zeroCounter = 0;
|
|
268900
|
+
const zeroes = [];
|
|
268901
|
+
for (i = 0;i < this.parsedAddress.length; i++) {
|
|
268902
|
+
const value = parseInt(this.parsedAddress[i], 16);
|
|
268903
|
+
if (value === 0) {
|
|
268904
|
+
zeroCounter++;
|
|
268905
|
+
}
|
|
268906
|
+
if (value !== 0 && zeroCounter > 0) {
|
|
268907
|
+
if (zeroCounter > 1) {
|
|
268908
|
+
zeroes.push([i - zeroCounter, i - 1]);
|
|
268909
|
+
}
|
|
268910
|
+
zeroCounter = 0;
|
|
268911
|
+
}
|
|
268912
|
+
}
|
|
268913
|
+
if (zeroCounter > 1) {
|
|
268914
|
+
zeroes.push([this.parsedAddress.length - zeroCounter, this.parsedAddress.length - 1]);
|
|
268915
|
+
}
|
|
268916
|
+
const zeroLengths = zeroes.map((n) => n[1] - n[0] + 1);
|
|
268917
|
+
if (zeroes.length > 0) {
|
|
268918
|
+
const index = zeroLengths.indexOf(Math.max(...zeroLengths));
|
|
268919
|
+
groups = compact(this.parsedAddress, zeroes[index]);
|
|
268920
|
+
} else {
|
|
268921
|
+
groups = this.parsedAddress;
|
|
268922
|
+
}
|
|
268923
|
+
for (i = 0;i < groups.length; i++) {
|
|
268924
|
+
if (groups[i] !== "compact") {
|
|
268925
|
+
groups[i] = parseInt(groups[i], 16).toString(16);
|
|
268926
|
+
}
|
|
268927
|
+
}
|
|
268928
|
+
let correct = groups.join(":");
|
|
268929
|
+
correct = correct.replace(/^compact$/, "::");
|
|
268930
|
+
correct = correct.replace(/(^compact)|(compact$)/, ":");
|
|
268931
|
+
correct = correct.replace(/compact/, "");
|
|
268932
|
+
return correct;
|
|
268933
|
+
}
|
|
268934
|
+
binaryZeroPad() {
|
|
268935
|
+
if (this._binaryZeroPad === undefined) {
|
|
268936
|
+
this._binaryZeroPad = this.bigInt().toString(2).padStart(constants6.BITS, "0");
|
|
268937
|
+
}
|
|
268938
|
+
return this._binaryZeroPad;
|
|
268939
|
+
}
|
|
268940
|
+
parse4in6(address) {
|
|
268941
|
+
if (address.indexOf(".") === -1) {
|
|
268942
|
+
return address;
|
|
268943
|
+
}
|
|
268944
|
+
const groups = address.split(":");
|
|
268945
|
+
const lastGroup = groups.slice(-1)[0];
|
|
268946
|
+
const v4Octets = lastGroup.split(".");
|
|
268947
|
+
if (v4Octets.length === constants4.GROUPS && v4Octets.every((octet) => /^\d{1,3}$/.test(octet))) {
|
|
268948
|
+
if (v4Octets.some((octet) => /^0\d/.test(octet))) {
|
|
268949
|
+
const highlighted = v4Octets.map(spanLeadingZeroes4).join(".");
|
|
268950
|
+
const prefix = groups.slice(0, -1).map(helpers.escapeHtml).join(":");
|
|
268951
|
+
const separator = groups.length > 1 ? ":" : "";
|
|
268952
|
+
throw new address_error_1.AddressError("IPv4 addresses can't have leading zeroes.", `${prefix}${separator}${highlighted}`);
|
|
268953
|
+
}
|
|
268954
|
+
}
|
|
268955
|
+
const address4 = lastGroup.match(constants4.RE_ADDRESS);
|
|
268956
|
+
if (address4) {
|
|
268957
|
+
this.parsedAddress4 = address4[0];
|
|
268958
|
+
const v4Suffix = this.subnetMask >= 96 ? `/${this.subnetMask - 96}` : "";
|
|
268959
|
+
this.address4 = new ipv4_1.Address4(`${this.parsedAddress4}${v4Suffix}`);
|
|
268960
|
+
this.v4 = true;
|
|
268961
|
+
groups[groups.length - 1] = this.address4.toGroup6();
|
|
268962
|
+
address = groups.join(":");
|
|
268963
|
+
}
|
|
268964
|
+
return address;
|
|
268965
|
+
}
|
|
268966
|
+
parse(address) {
|
|
268967
|
+
address = this.parse4in6(address);
|
|
268968
|
+
const badCharacters = address.match(constants6.RE_BAD_CHARACTERS);
|
|
268969
|
+
if (badCharacters) {
|
|
268970
|
+
throw new address_error_1.AddressError(`Bad character${badCharacters.length > 1 ? "s" : ""} detected in address: ${badCharacters.join("")}`, address.replace(constants6.RE_BAD_CHARACTERS, '<span class="parse-error">$1</span>'));
|
|
268971
|
+
}
|
|
268972
|
+
const badAddress = address.match(constants6.RE_BAD_ADDRESS);
|
|
268973
|
+
if (badAddress) {
|
|
268974
|
+
throw new address_error_1.AddressError(`Address failed regex: ${badAddress.join("")}`, address.replace(constants6.RE_BAD_ADDRESS, '<span class="parse-error">$1</span>'));
|
|
268975
|
+
}
|
|
268976
|
+
let groups = [];
|
|
268977
|
+
const halves = address.split("::");
|
|
268978
|
+
if (halves.length === 2) {
|
|
268979
|
+
let first2 = halves[0].split(":");
|
|
268980
|
+
let last2 = halves[1].split(":");
|
|
268981
|
+
if (first2.length === 1 && first2[0] === "") {
|
|
268982
|
+
first2 = [];
|
|
268983
|
+
}
|
|
268984
|
+
if (last2.length === 1 && last2[0] === "") {
|
|
268985
|
+
last2 = [];
|
|
268986
|
+
}
|
|
268987
|
+
const remaining = this.groups - (first2.length + last2.length);
|
|
268988
|
+
if (!remaining) {
|
|
268989
|
+
throw new address_error_1.AddressError("Error parsing groups");
|
|
268990
|
+
}
|
|
268991
|
+
this.elidedGroups = remaining;
|
|
268992
|
+
this.elisionBegin = first2.length;
|
|
268993
|
+
this.elisionEnd = first2.length + this.elidedGroups;
|
|
268994
|
+
groups = groups.concat(first2);
|
|
268995
|
+
for (let i = 0;i < remaining; i++) {
|
|
268996
|
+
groups.push("0");
|
|
268997
|
+
}
|
|
268998
|
+
groups = groups.concat(last2);
|
|
268999
|
+
} else if (halves.length === 1) {
|
|
269000
|
+
groups = address.split(":");
|
|
269001
|
+
this.elidedGroups = 0;
|
|
269002
|
+
} else {
|
|
269003
|
+
throw new address_error_1.AddressError("Too many :: groups found");
|
|
269004
|
+
}
|
|
269005
|
+
groups = groups.map((group) => parseInt(group, 16).toString(16));
|
|
269006
|
+
if (groups.length !== this.groups) {
|
|
269007
|
+
throw new address_error_1.AddressError("Incorrect number of groups found");
|
|
269008
|
+
}
|
|
269009
|
+
return groups;
|
|
269010
|
+
}
|
|
269011
|
+
canonicalForm() {
|
|
269012
|
+
return this.parsedAddress.map(paddedHex).join(":");
|
|
269013
|
+
}
|
|
269014
|
+
decimal() {
|
|
269015
|
+
return this.parsedAddress.map((n) => parseInt(n, 16).toString(10).padStart(5, "0")).join(":");
|
|
269016
|
+
}
|
|
269017
|
+
bigInt() {
|
|
269018
|
+
return BigInt(`0x${this.parsedAddress.map(paddedHex).join("")}`);
|
|
269019
|
+
}
|
|
269020
|
+
to4() {
|
|
269021
|
+
const binary = this.binaryZeroPad().split("");
|
|
269022
|
+
const hex = BigInt(`0b${binary.slice(96, 128).join("")}`).toString(16).padStart(8, "0");
|
|
269023
|
+
if (this.subnetMask >= 96) {
|
|
269024
|
+
const v4Mask = this.subnetMask - 96;
|
|
269025
|
+
const groups = [];
|
|
269026
|
+
for (let i = 0;i < 8; i += 2) {
|
|
269027
|
+
groups.push(parseInt(hex.slice(i, i + 2), 16));
|
|
269028
|
+
}
|
|
269029
|
+
return new ipv4_1.Address4(`${groups.join(".")}/${v4Mask}`);
|
|
269030
|
+
}
|
|
269031
|
+
return ipv4_1.Address4.fromHex(hex);
|
|
269032
|
+
}
|
|
269033
|
+
to4in6() {
|
|
269034
|
+
const address4 = this.to4();
|
|
269035
|
+
const address6 = new Address6(this.parsedAddress.slice(0, 6).join(":"), 6);
|
|
269036
|
+
const correct = address6.correctForm();
|
|
269037
|
+
let infix = "";
|
|
269038
|
+
if (!/:$/.test(correct)) {
|
|
269039
|
+
infix = ":";
|
|
269040
|
+
}
|
|
269041
|
+
return correct + infix + address4.correctForm();
|
|
269042
|
+
}
|
|
269043
|
+
inspectTeredo() {
|
|
269044
|
+
const prefix = this.getBitsBase16(0, 32);
|
|
269045
|
+
const bitsForUdpPort = this.getBits(80, 96);
|
|
269046
|
+
const udpPort = (bitsForUdpPort ^ BigInt("0xffff")).toString();
|
|
269047
|
+
const server4 = ipv4_1.Address4.fromHex(this.getBitsBase16(32, 64));
|
|
269048
|
+
const bitsForClient4 = this.getBits(96, 128);
|
|
269049
|
+
const client4 = ipv4_1.Address4.fromHex((bitsForClient4 ^ BigInt("0xffffffff")).toString(16).padStart(8, "0"));
|
|
269050
|
+
const flagsBase2 = this.getBitsBase2(64, 80);
|
|
269051
|
+
const coneNat = (0, common_1.testBit)(flagsBase2, 15);
|
|
269052
|
+
const reserved = (0, common_1.testBit)(flagsBase2, 14);
|
|
269053
|
+
const groupIndividual = (0, common_1.testBit)(flagsBase2, 8);
|
|
269054
|
+
const universalLocal = (0, common_1.testBit)(flagsBase2, 9);
|
|
269055
|
+
const nonce = BigInt(`0b${flagsBase2.slice(2, 6) + flagsBase2.slice(8, 16)}`).toString(10);
|
|
269056
|
+
return {
|
|
269057
|
+
prefix: `${prefix.slice(0, 4)}:${prefix.slice(4, 8)}`,
|
|
269058
|
+
server4: server4.address,
|
|
269059
|
+
client4: client4.address,
|
|
269060
|
+
flags: flagsBase2,
|
|
269061
|
+
coneNat,
|
|
269062
|
+
microsoft: {
|
|
269063
|
+
reserved,
|
|
269064
|
+
universalLocal,
|
|
269065
|
+
groupIndividual,
|
|
269066
|
+
nonce
|
|
269067
|
+
},
|
|
269068
|
+
udpPort
|
|
269069
|
+
};
|
|
269070
|
+
}
|
|
269071
|
+
inspect6to4() {
|
|
269072
|
+
const prefix = this.getBitsBase16(0, 16);
|
|
269073
|
+
const gateway = ipv4_1.Address4.fromHex(this.getBitsBase16(16, 48));
|
|
269074
|
+
return {
|
|
269075
|
+
prefix: prefix.slice(0, 4),
|
|
269076
|
+
gateway: gateway.address
|
|
269077
|
+
};
|
|
269078
|
+
}
|
|
269079
|
+
to6to4() {
|
|
269080
|
+
if (!this.is4()) {
|
|
269081
|
+
return null;
|
|
269082
|
+
}
|
|
269083
|
+
const addr6to4 = [
|
|
269084
|
+
"2002",
|
|
269085
|
+
this.getBitsBase16(96, 112),
|
|
269086
|
+
this.getBitsBase16(112, 128),
|
|
269087
|
+
"",
|
|
269088
|
+
"/16"
|
|
269089
|
+
].join(":");
|
|
269090
|
+
return new Address6(addr6to4);
|
|
269091
|
+
}
|
|
269092
|
+
static fromAddress4Nat64(address, prefix = "64:ff9b::/96") {
|
|
269093
|
+
const v4 = new ipv4_1.Address4(address);
|
|
269094
|
+
const prefix6 = new Address6(prefix);
|
|
269095
|
+
const pl2 = prefix6.subnetMask;
|
|
269096
|
+
if (pl2 !== 32 && pl2 !== 40 && pl2 !== 48 && pl2 !== 56 && pl2 !== 64 && pl2 !== 96) {
|
|
269097
|
+
throw new address_error_1.AddressError("NAT64 prefix length must be 32, 40, 48, 56, 64, or 96");
|
|
269098
|
+
}
|
|
269099
|
+
const prefixBits = prefix6.binaryZeroPad();
|
|
269100
|
+
const v4Bits = v4.binaryZeroPad();
|
|
269101
|
+
let bits;
|
|
269102
|
+
if (pl2 === 96) {
|
|
269103
|
+
bits = prefixBits.slice(0, 96) + v4Bits;
|
|
269104
|
+
} else {
|
|
269105
|
+
const beforeU = 64 - pl2;
|
|
269106
|
+
bits = [
|
|
269107
|
+
prefixBits.slice(0, pl2),
|
|
269108
|
+
v4Bits.slice(0, beforeU),
|
|
269109
|
+
"00000000",
|
|
269110
|
+
v4Bits.slice(beforeU),
|
|
269111
|
+
"0".repeat(128 - 72 - (32 - beforeU))
|
|
269112
|
+
].join("");
|
|
269113
|
+
}
|
|
269114
|
+
const hex = BigInt(`0b${bits}`).toString(16).padStart(32, "0");
|
|
269115
|
+
const groups = [];
|
|
269116
|
+
for (let i = 0;i < 8; i++) {
|
|
269117
|
+
groups.push(hex.slice(i * 4, (i + 1) * 4));
|
|
269118
|
+
}
|
|
269119
|
+
return new Address6(groups.join(":"));
|
|
269120
|
+
}
|
|
269121
|
+
toAddress4Nat64(prefix = "64:ff9b::/96") {
|
|
269122
|
+
const prefix6 = new Address6(prefix);
|
|
269123
|
+
const pl2 = prefix6.subnetMask;
|
|
269124
|
+
if (pl2 !== 32 && pl2 !== 40 && pl2 !== 48 && pl2 !== 56 && pl2 !== 64 && pl2 !== 96) {
|
|
269125
|
+
throw new address_error_1.AddressError("NAT64 prefix length must be 32, 40, 48, 56, 64, or 96");
|
|
269126
|
+
}
|
|
269127
|
+
if (!this.isHostInSubnet(prefix6)) {
|
|
269128
|
+
return null;
|
|
269129
|
+
}
|
|
269130
|
+
const bits = this.binaryZeroPad();
|
|
269131
|
+
let v4Bits;
|
|
269132
|
+
if (pl2 === 96) {
|
|
269133
|
+
v4Bits = bits.slice(96, 128);
|
|
269134
|
+
} else {
|
|
269135
|
+
const beforeU = 64 - pl2;
|
|
269136
|
+
v4Bits = bits.slice(pl2, pl2 + beforeU) + bits.slice(72, 72 + (32 - beforeU));
|
|
269137
|
+
}
|
|
269138
|
+
const octets = [];
|
|
269139
|
+
for (let i = 0;i < 4; i++) {
|
|
269140
|
+
octets.push(parseInt(v4Bits.slice(i * 8, (i + 1) * 8), 2).toString());
|
|
269141
|
+
}
|
|
269142
|
+
return new ipv4_1.Address4(octets.join("."));
|
|
269143
|
+
}
|
|
269144
|
+
toByteArray() {
|
|
269145
|
+
const value = this.bigInt().toString(16).padStart(constants6.BITS / 4, "0");
|
|
269146
|
+
const bytes = [];
|
|
269147
|
+
for (let i = 0, length = value.length;i < length; i += 2) {
|
|
269148
|
+
bytes.push(parseInt(value.substring(i, i + 2), 16));
|
|
269149
|
+
}
|
|
269150
|
+
return bytes;
|
|
269151
|
+
}
|
|
269152
|
+
toUnsignedByteArray() {
|
|
269153
|
+
return this.toByteArray().map(unsignByte);
|
|
269154
|
+
}
|
|
269155
|
+
static fromByteArray(bytes) {
|
|
269156
|
+
common.assertByteArray(bytes, 16, "IPv6", -128);
|
|
269157
|
+
return this.fromUnsignedByteArray(bytes.map(unsignByte));
|
|
269158
|
+
}
|
|
269159
|
+
static fromUnsignedByteArray(bytes) {
|
|
269160
|
+
common.assertByteArray(bytes, 16, "IPv6", 0);
|
|
269161
|
+
const BYTE_MAX = BigInt("256");
|
|
269162
|
+
let result = BigInt("0");
|
|
269163
|
+
let multiplier = BigInt("1");
|
|
269164
|
+
for (let i = bytes.length - 1;i >= 0; i--) {
|
|
269165
|
+
result += multiplier * BigInt(bytes[i].toString(10));
|
|
269166
|
+
multiplier *= BYTE_MAX;
|
|
269167
|
+
}
|
|
269168
|
+
return Address6.fromBigInt(result);
|
|
269169
|
+
}
|
|
269170
|
+
isCanonical() {
|
|
269171
|
+
return this.addressMinusSuffix === this.canonicalForm();
|
|
269172
|
+
}
|
|
269173
|
+
isLinkLocal() {
|
|
269174
|
+
const embedded = this.embeddedIPv4();
|
|
269175
|
+
if (embedded) {
|
|
269176
|
+
return embedded.isLinkLocal();
|
|
269177
|
+
}
|
|
269178
|
+
if (this.getBitsBase2(0, 64) === "1111111010000000000000000000000000000000000000000000000000000000") {
|
|
269179
|
+
return true;
|
|
269180
|
+
}
|
|
269181
|
+
return false;
|
|
269182
|
+
}
|
|
269183
|
+
isMulticast() {
|
|
269184
|
+
const embedded = this.embeddedIPv4();
|
|
269185
|
+
if (embedded) {
|
|
269186
|
+
return embedded.isMulticast();
|
|
269187
|
+
}
|
|
269188
|
+
const type = this.getType();
|
|
269189
|
+
return type === "Multicast" || type.startsWith("Multicast ");
|
|
269190
|
+
}
|
|
269191
|
+
is4() {
|
|
269192
|
+
return this.v4;
|
|
269193
|
+
}
|
|
269194
|
+
isMapped4() {
|
|
269195
|
+
return this.isHostInSubnet(IPV4_MAPPED_SUBNET);
|
|
269196
|
+
}
|
|
269197
|
+
embeddedIPv4() {
|
|
269198
|
+
if (this.isMapped4() || this.isHostInSubnet(NAT64_WELL_KNOWN_SUBNET)) {
|
|
269199
|
+
return this.to4();
|
|
269200
|
+
}
|
|
269201
|
+
return null;
|
|
269202
|
+
}
|
|
269203
|
+
isTeredo() {
|
|
269204
|
+
return this.isHostInSubnet(TEREDO_SUBNET);
|
|
269205
|
+
}
|
|
269206
|
+
is6to4() {
|
|
269207
|
+
return this.isHostInSubnet(SIX_TO_FOUR_SUBNET);
|
|
269208
|
+
}
|
|
269209
|
+
isLoopback() {
|
|
269210
|
+
const embedded = this.embeddedIPv4();
|
|
269211
|
+
if (embedded) {
|
|
269212
|
+
return embedded.isLoopback();
|
|
269213
|
+
}
|
|
269214
|
+
return this.getType() === "Loopback";
|
|
269215
|
+
}
|
|
269216
|
+
isULA() {
|
|
269217
|
+
return this.isHostInSubnet(ULA_SUBNET);
|
|
269218
|
+
}
|
|
269219
|
+
isPrivate() {
|
|
269220
|
+
const embedded = this.embeddedIPv4();
|
|
269221
|
+
if (embedded) {
|
|
269222
|
+
return embedded.isPrivate();
|
|
269223
|
+
}
|
|
269224
|
+
return this.isULA();
|
|
269225
|
+
}
|
|
269226
|
+
isCGNAT() {
|
|
269227
|
+
const embedded = this.embeddedIPv4();
|
|
269228
|
+
if (embedded) {
|
|
269229
|
+
return embedded.isCGNAT();
|
|
269230
|
+
}
|
|
269231
|
+
return false;
|
|
269232
|
+
}
|
|
269233
|
+
isBroadcast() {
|
|
269234
|
+
const embedded = this.embeddedIPv4();
|
|
269235
|
+
if (embedded) {
|
|
269236
|
+
return embedded.isBroadcast();
|
|
269237
|
+
}
|
|
269238
|
+
return false;
|
|
269239
|
+
}
|
|
269240
|
+
isUnspecified() {
|
|
269241
|
+
const embedded = this.embeddedIPv4();
|
|
269242
|
+
if (embedded) {
|
|
269243
|
+
return embedded.isUnspecified();
|
|
269244
|
+
}
|
|
269245
|
+
return this.getType() === "Unspecified";
|
|
269246
|
+
}
|
|
269247
|
+
isDocumentation() {
|
|
269248
|
+
return this.isHostInSubnet(DOCUMENTATION_SUBNET);
|
|
269249
|
+
}
|
|
269250
|
+
href(optionalPort) {
|
|
269251
|
+
if (optionalPort === undefined) {
|
|
269252
|
+
optionalPort = "";
|
|
269253
|
+
} else {
|
|
269254
|
+
optionalPort = `:${optionalPort}`;
|
|
269255
|
+
}
|
|
269256
|
+
return `http://[${this.correctForm()}]${optionalPort}/`;
|
|
269257
|
+
}
|
|
269258
|
+
link(options) {
|
|
269259
|
+
if (!options) {
|
|
269260
|
+
options = {};
|
|
269261
|
+
}
|
|
269262
|
+
if (options.className === undefined) {
|
|
269263
|
+
options.className = "";
|
|
269264
|
+
}
|
|
269265
|
+
if (options.prefix === undefined) {
|
|
269266
|
+
options.prefix = "/#address=";
|
|
269267
|
+
}
|
|
269268
|
+
if (options.v4 === undefined) {
|
|
269269
|
+
options.v4 = false;
|
|
269270
|
+
}
|
|
269271
|
+
let formFunction = this.correctForm;
|
|
269272
|
+
if (options.v4) {
|
|
269273
|
+
formFunction = this.to4in6;
|
|
269274
|
+
}
|
|
269275
|
+
const form = formFunction.call(this);
|
|
269276
|
+
const safeHref = helpers.escapeHtml(`${options.prefix}${form}`);
|
|
269277
|
+
const safeForm = helpers.escapeHtml(form);
|
|
269278
|
+
if (options.className) {
|
|
269279
|
+
const safeClass = helpers.escapeHtml(options.className);
|
|
269280
|
+
return `<a href="${safeHref}" class="${safeClass}">${safeForm}</a>`;
|
|
269281
|
+
}
|
|
269282
|
+
return `<a href="${safeHref}">${safeForm}</a>`;
|
|
269283
|
+
}
|
|
269284
|
+
group() {
|
|
269285
|
+
if (this.elidedGroups === 0) {
|
|
269286
|
+
return helpers.simpleGroup(this.addressMinusSuffix).join(":");
|
|
269287
|
+
}
|
|
269288
|
+
assert(typeof this.elidedGroups === "number");
|
|
269289
|
+
assert(typeof this.elisionBegin === "number");
|
|
269290
|
+
const output = [];
|
|
269291
|
+
const [left, right] = this.addressMinusSuffix.split("::");
|
|
269292
|
+
if (left.length) {
|
|
269293
|
+
output.push(...helpers.simpleGroup(left));
|
|
269294
|
+
} else {
|
|
269295
|
+
output.push("");
|
|
269296
|
+
}
|
|
269297
|
+
const classes = ["hover-group"];
|
|
269298
|
+
for (let i = this.elisionBegin;i < this.elisionBegin + this.elidedGroups; i++) {
|
|
269299
|
+
classes.push(`group-${i}`);
|
|
269300
|
+
}
|
|
269301
|
+
output.push(`<span class="${classes.join(" ")}"></span>`);
|
|
269302
|
+
if (right.length) {
|
|
269303
|
+
output.push(...helpers.simpleGroup(right, this.elisionEnd));
|
|
269304
|
+
} else {
|
|
269305
|
+
output.push("");
|
|
269306
|
+
}
|
|
269307
|
+
if (this.is4()) {
|
|
269308
|
+
assert(this.address4 instanceof ipv4_1.Address4);
|
|
269309
|
+
output.pop();
|
|
269310
|
+
output.push(this.address4.groupForV6());
|
|
269311
|
+
}
|
|
269312
|
+
return output.join(":");
|
|
269313
|
+
}
|
|
269314
|
+
regularExpressionString(substringSearch = false) {
|
|
269315
|
+
let output = [];
|
|
269316
|
+
const address6 = new Address6(this.correctForm());
|
|
269317
|
+
if (address6.elidedGroups === 0) {
|
|
269318
|
+
output.push((0, regular_expressions_1.simpleRegularExpression)(address6.parsedAddress));
|
|
269319
|
+
} else if (address6.elidedGroups === constants6.GROUPS) {
|
|
269320
|
+
output.push((0, regular_expressions_1.possibleElisions)(constants6.GROUPS));
|
|
269321
|
+
} else {
|
|
269322
|
+
const halves = address6.address.split("::");
|
|
269323
|
+
if (halves[0].length) {
|
|
269324
|
+
output.push((0, regular_expressions_1.simpleRegularExpression)(halves[0].split(":")));
|
|
269325
|
+
}
|
|
269326
|
+
assert(typeof address6.elidedGroups === "number");
|
|
269327
|
+
output.push((0, regular_expressions_1.possibleElisions)(address6.elidedGroups, halves[0].length !== 0, halves[1].length !== 0));
|
|
269328
|
+
if (halves[1].length) {
|
|
269329
|
+
output.push((0, regular_expressions_1.simpleRegularExpression)(halves[1].split(":")));
|
|
269330
|
+
}
|
|
269331
|
+
output = [output.join(":")];
|
|
269332
|
+
}
|
|
269333
|
+
if (!substringSearch) {
|
|
269334
|
+
output = [
|
|
269335
|
+
"(?=^|",
|
|
269336
|
+
regular_expressions_1.ADDRESS_BOUNDARY,
|
|
269337
|
+
"|[^\\w\\:])(",
|
|
269338
|
+
...output,
|
|
269339
|
+
")(?=[^\\w\\:]|",
|
|
269340
|
+
regular_expressions_1.ADDRESS_BOUNDARY,
|
|
269341
|
+
"|$)"
|
|
269342
|
+
];
|
|
269343
|
+
}
|
|
269344
|
+
return output.join("");
|
|
269345
|
+
}
|
|
269346
|
+
regularExpression(substringSearch = false) {
|
|
269347
|
+
return new RegExp(this.regularExpressionString(substringSearch), "i");
|
|
269348
|
+
}
|
|
269349
|
+
}
|
|
269350
|
+
exports.Address6 = Address6;
|
|
269351
|
+
var TYPE_SUBNETS = Object.keys(constants6.TYPES).map((subnet) => [
|
|
269352
|
+
new Address6(subnet),
|
|
269353
|
+
constants6.TYPES[subnet]
|
|
269354
|
+
]);
|
|
269355
|
+
var TEREDO_SUBNET = new Address6("2001::/32");
|
|
269356
|
+
var SIX_TO_FOUR_SUBNET = new Address6("2002::/16");
|
|
269357
|
+
var ULA_SUBNET = new Address6("fc00::/7");
|
|
269358
|
+
var DOCUMENTATION_SUBNET = new Address6("2001:db8::/32");
|
|
269359
|
+
var IPV4_MAPPED_SUBNET = new Address6("::ffff:0:0/96");
|
|
269360
|
+
var NAT64_WELL_KNOWN_SUBNET = new Address6("64:ff9b::/96");
|
|
269361
|
+
});
|
|
269362
|
+
|
|
269363
|
+
// ../../node_modules/ip-address/dist/ip-address.js
|
|
269364
|
+
var require_ip_address = __commonJS((exports) => {
|
|
269365
|
+
var __createBinding = exports && exports.__createBinding || (Object.create ? function(o, m, k, k2) {
|
|
269366
|
+
if (k2 === undefined)
|
|
269367
|
+
k2 = k;
|
|
269368
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
269369
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
269370
|
+
desc = { enumerable: true, get: function() {
|
|
269371
|
+
return m[k];
|
|
269372
|
+
} };
|
|
269373
|
+
}
|
|
269374
|
+
Object.defineProperty(o, k2, desc);
|
|
269375
|
+
} : function(o, m, k, k2) {
|
|
269376
|
+
if (k2 === undefined)
|
|
269377
|
+
k2 = k;
|
|
269378
|
+
o[k2] = m[k];
|
|
269379
|
+
});
|
|
269380
|
+
var __setModuleDefault = exports && exports.__setModuleDefault || (Object.create ? function(o, v) {
|
|
269381
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
269382
|
+
} : function(o, v) {
|
|
269383
|
+
o["default"] = v;
|
|
269384
|
+
});
|
|
269385
|
+
var __importStar = exports && exports.__importStar || function(mod2) {
|
|
269386
|
+
if (mod2 && mod2.__esModule)
|
|
269387
|
+
return mod2;
|
|
269388
|
+
var result = {};
|
|
269389
|
+
if (mod2 != null) {
|
|
269390
|
+
for (var k in mod2)
|
|
269391
|
+
if (k !== "default" && Object.prototype.hasOwnProperty.call(mod2, k))
|
|
269392
|
+
__createBinding(result, mod2, k);
|
|
269393
|
+
}
|
|
269394
|
+
__setModuleDefault(result, mod2);
|
|
269395
|
+
return result;
|
|
269396
|
+
};
|
|
269397
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
269398
|
+
exports.v6 = exports.AddressError = exports.Address6 = exports.Address4 = undefined;
|
|
269399
|
+
var ipv4_1 = require_ipv4();
|
|
269400
|
+
Object.defineProperty(exports, "Address4", { enumerable: true, get: function() {
|
|
269401
|
+
return ipv4_1.Address4;
|
|
269402
|
+
} });
|
|
269403
|
+
var ipv6_1 = require_ipv6();
|
|
269404
|
+
Object.defineProperty(exports, "Address6", { enumerable: true, get: function() {
|
|
269405
|
+
return ipv6_1.Address6;
|
|
269406
|
+
} });
|
|
269407
|
+
var address_error_1 = require_address_error();
|
|
269408
|
+
Object.defineProperty(exports, "AddressError", { enumerable: true, get: function() {
|
|
269409
|
+
return address_error_1.AddressError;
|
|
269410
|
+
} });
|
|
269411
|
+
var helpers = __importStar(require_helpers2());
|
|
269412
|
+
exports.v6 = { helpers };
|
|
269413
|
+
});
|
|
269414
|
+
|
|
267990
269415
|
// src/instrumentation.ts
|
|
267991
269416
|
init_logger();
|
|
267992
269417
|
init_telemetry();
|
|
@@ -282162,10 +283587,15 @@ class IncrementalLedgerRepository {
|
|
|
282162
283587
|
constructor(db) {
|
|
282163
283588
|
this.db = db;
|
|
282164
283589
|
}
|
|
282165
|
-
async get(environmentId,
|
|
283590
|
+
async get(environmentId, table) {
|
|
282166
283591
|
const row = await this.db.get(`SELECT * FROM incremental_ledger
|
|
282167
283592
|
WHERE environment_id = ? AND connection_name = ?
|
|
282168
|
-
AND physical_table_name = ?`, [
|
|
283593
|
+
AND storage_destination_name = ? AND physical_table_name = ?`, [
|
|
283594
|
+
environmentId,
|
|
283595
|
+
table.connectionName,
|
|
283596
|
+
table.storageDestinationName ?? "",
|
|
283597
|
+
table.physicalTableName
|
|
283598
|
+
]);
|
|
282169
283599
|
return row ? mapRow(row) : null;
|
|
282170
283600
|
}
|
|
282171
283601
|
async upsert(entry) {
|
|
@@ -282174,10 +283604,11 @@ class IncrementalLedgerRepository {
|
|
|
282174
283604
|
environment_id, package_name, source_entity_id,
|
|
282175
283605
|
covered_through_value, covered_through_type,
|
|
282176
283606
|
watermark_dimension, merge_key_dimensions, derived_strategy,
|
|
282177
|
-
physical_table_name, connection_name,
|
|
283607
|
+
physical_table_name, connection_name, storage_destination_name,
|
|
282178
283608
|
advanced_by_materialization_id, advanced_at, created_at
|
|
282179
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
282180
|
-
ON CONFLICT (environment_id, connection_name,
|
|
283609
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
283610
|
+
ON CONFLICT (environment_id, connection_name,
|
|
283611
|
+
storage_destination_name, physical_table_name)
|
|
282181
283612
|
DO UPDATE SET
|
|
282182
283613
|
package_name = EXCLUDED.package_name,
|
|
282183
283614
|
source_entity_id = EXCLUDED.source_entity_id,
|
|
@@ -282186,6 +283617,7 @@ class IncrementalLedgerRepository {
|
|
|
282186
283617
|
watermark_dimension = EXCLUDED.watermark_dimension,
|
|
282187
283618
|
merge_key_dimensions = EXCLUDED.merge_key_dimensions,
|
|
282188
283619
|
derived_strategy = EXCLUDED.derived_strategy,
|
|
283620
|
+
storage_destination_name = EXCLUDED.storage_destination_name,
|
|
282189
283621
|
advanced_by_materialization_id = EXCLUDED.advanced_by_materialization_id,
|
|
282190
283622
|
advanced_at = EXCLUDED.advanced_at
|
|
282191
283623
|
RETURNING *`, [
|
|
@@ -282199,16 +283631,22 @@ class IncrementalLedgerRepository {
|
|
|
282199
283631
|
entry.derivedStrategy,
|
|
282200
283632
|
entry.physicalTableName,
|
|
282201
283633
|
entry.connectionName,
|
|
283634
|
+
entry.storageDestinationName ?? "",
|
|
282202
283635
|
entry.advancedByMaterializationId,
|
|
282203
283636
|
now,
|
|
282204
283637
|
now
|
|
282205
283638
|
]);
|
|
282206
283639
|
return mapRow(rows[0]);
|
|
282207
283640
|
}
|
|
282208
|
-
async deleteEntry(environmentId,
|
|
283641
|
+
async deleteEntry(environmentId, table) {
|
|
282209
283642
|
await this.db.run(`DELETE FROM incremental_ledger
|
|
282210
283643
|
WHERE environment_id = ? AND connection_name = ?
|
|
282211
|
-
AND physical_table_name = ?`, [
|
|
283644
|
+
AND storage_destination_name = ? AND physical_table_name = ?`, [
|
|
283645
|
+
environmentId,
|
|
283646
|
+
table.connectionName,
|
|
283647
|
+
table.storageDestinationName ?? "",
|
|
283648
|
+
table.physicalTableName
|
|
283649
|
+
]);
|
|
282212
283650
|
}
|
|
282213
283651
|
async deleteByEnvironmentId(environmentId) {
|
|
282214
283652
|
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ?", [environmentId]);
|
|
@@ -282229,6 +283667,7 @@ function mapRow(row) {
|
|
|
282229
283667
|
derivedStrategy: row.derived_strategy,
|
|
282230
283668
|
physicalTableName: row.physical_table_name,
|
|
282231
283669
|
connectionName: row.connection_name,
|
|
283670
|
+
storageDestinationName: typeof row.storage_destination_name === "string" && row.storage_destination_name.length > 0 ? row.storage_destination_name : undefined,
|
|
282232
283671
|
advancedByMaterializationId: row.advanced_by_materialization_id != null ? row.advanced_by_materialization_id : null,
|
|
282233
283672
|
advancedAt: new Date(row.advanced_at),
|
|
282234
283673
|
createdAt: new Date(row.created_at)
|
|
@@ -282716,14 +284155,14 @@ class DuckDBRepository {
|
|
|
282716
284155
|
async deleteMaterialization(id) {
|
|
282717
284156
|
return this.materializationRepo.deleteById(id);
|
|
282718
284157
|
}
|
|
282719
|
-
async getIncrementalLedgerEntry(environmentId,
|
|
282720
|
-
return this.incrementalLedgerRepo.get(environmentId,
|
|
284158
|
+
async getIncrementalLedgerEntry(environmentId, table) {
|
|
284159
|
+
return this.incrementalLedgerRepo.get(environmentId, table);
|
|
282721
284160
|
}
|
|
282722
284161
|
async upsertIncrementalLedgerEntry(entry) {
|
|
282723
284162
|
return this.incrementalLedgerRepo.upsert(entry);
|
|
282724
284163
|
}
|
|
282725
|
-
async deleteIncrementalLedgerEntry(environmentId,
|
|
282726
|
-
return this.incrementalLedgerRepo.deleteEntry(environmentId,
|
|
284164
|
+
async deleteIncrementalLedgerEntry(environmentId, table) {
|
|
284165
|
+
return this.incrementalLedgerRepo.deleteEntry(environmentId, table);
|
|
282727
284166
|
}
|
|
282728
284167
|
}
|
|
282729
284168
|
|
|
@@ -282739,6 +284178,7 @@ async function initializeSchema(db, force = false) {
|
|
|
282739
284178
|
logger.info("Creating database schema for the first time...");
|
|
282740
284179
|
}
|
|
282741
284180
|
await dropPackageKeyedIncrementalLedger(db);
|
|
284181
|
+
await dropStoreBlindIncrementalLedger(db);
|
|
282742
284182
|
await createDeclaredTables(db);
|
|
282743
284183
|
if (initialized) {
|
|
282744
284184
|
await reconcileDeclaredColumns(db);
|
|
@@ -282826,10 +284266,34 @@ async function createDeclaredTables(db) {
|
|
|
282826
284266
|
derived_strategy VARCHAR NOT NULL,
|
|
282827
284267
|
physical_table_name VARCHAR NOT NULL,
|
|
282828
284268
|
connection_name VARCHAR NOT NULL,
|
|
284269
|
+
-- The storage destination the table lives in, or '' when it lives in
|
|
284270
|
+
-- connection_name's own warehouse (the colocated default). Part of the
|
|
284271
|
+
-- table's IDENTITY, not a description of it: a destination and a connection
|
|
284272
|
+
-- are separate namespaces that may share a name, and a source's physical
|
|
284273
|
+
-- name does not change when it moves between them, so without this a
|
|
284274
|
+
-- boundary measured on a stored table is indistinguishable from one
|
|
284275
|
+
-- measured on a colocated table of the same name.
|
|
284276
|
+
--
|
|
284277
|
+
-- In the KEY, because two such tables coexist legitimately. A source
|
|
284278
|
+
-- persisted colocated and another persisted into a destination under the
|
|
284279
|
+
-- same name are two different tables and neither is a misconfiguration —
|
|
284280
|
+
-- the within-package collision check keys on the destination too, so it
|
|
284281
|
+
-- correctly says nothing about the pair. Sharing one row between them makes
|
|
284282
|
+
-- both seed forever: each read matches on the store, finds nothing, and
|
|
284283
|
+
-- overwrites the other's row on the way out. Nothing reports that; it just
|
|
284284
|
+
-- never advances.
|
|
284285
|
+
--
|
|
284286
|
+
-- '' rather than NULL, and that is what makes it keyable. A NULL in the key
|
|
284287
|
+
-- defeats ON CONFLICT (NULL never equals NULL, so two colocated rows both
|
|
284288
|
+
-- insert) and forces every read to compare with IS NOT DISTINCT FROM. The
|
|
284289
|
+
-- sentinel keeps the key a plain equality on every path.
|
|
284290
|
+
storage_destination_name VARCHAR NOT NULL DEFAULT '',
|
|
282829
284291
|
advanced_by_materialization_id VARCHAR,
|
|
282830
284292
|
advanced_at TIMESTAMP NOT NULL,
|
|
282831
284293
|
created_at TIMESTAMP NOT NULL,
|
|
282832
|
-
PRIMARY KEY (
|
|
284294
|
+
PRIMARY KEY (
|
|
284295
|
+
environment_id, connection_name, storage_destination_name,
|
|
284296
|
+
physical_table_name)
|
|
282833
284297
|
)
|
|
282834
284298
|
`);
|
|
282835
284299
|
await db.run(`
|
|
@@ -282983,6 +284447,19 @@ async function dropPackageKeyedIncrementalLedger(db) {
|
|
|
282983
284447
|
logger.info("Re-keying the incremental ledger onto (environment, connection, table); " + "recorded covered_through boundaries are discarded, so each incremental " + "source rebuilds in full once and then resumes advancing by delta");
|
|
282984
284448
|
await db.run("DROP TABLE IF EXISTS incremental_ledger");
|
|
282985
284449
|
}
|
|
284450
|
+
async function dropStoreBlindIncrementalLedger(db) {
|
|
284451
|
+
const present = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='incremental_ledger'");
|
|
284452
|
+
if (!present || present.length === 0) {
|
|
284453
|
+
return;
|
|
284454
|
+
}
|
|
284455
|
+
const columns = await db.all("PRAGMA table_info('incremental_ledger')");
|
|
284456
|
+
const destinationInKey = columns.some((column) => column.name === "storage_destination_name" && Boolean(column.pk));
|
|
284457
|
+
if (destinationInKey) {
|
|
284458
|
+
return;
|
|
284459
|
+
}
|
|
284460
|
+
logger.info("Re-keying the incremental ledger onto (environment, connection, store, " + "table); recorded covered_through boundaries are discarded, so each " + "incremental source rebuilds in full once and then resumes advancing by " + "delta");
|
|
284461
|
+
await db.run("DROP TABLE IF EXISTS incremental_ledger");
|
|
284462
|
+
}
|
|
282986
284463
|
async function createEntityEmbeddingsTable(db) {
|
|
282987
284464
|
await db.run(`
|
|
282988
284465
|
CREATE TABLE IF NOT EXISTS entity_embeddings (
|
|
@@ -283301,15 +284778,22 @@ init_logger();
|
|
|
283301
284778
|
init_utils4();
|
|
283302
284779
|
init_quoting();
|
|
283303
284780
|
import { decodeDottedTablePath } from "@malloydata/malloy";
|
|
283304
|
-
var
|
|
284781
|
+
var INCREMENTAL_SOURCE_RANGE_DIALECTS = new Set([
|
|
283305
284782
|
"postgres",
|
|
283306
284783
|
"snowflake",
|
|
283307
284784
|
"standardsql"
|
|
283308
284785
|
]);
|
|
284786
|
+
var INCREMENTAL_TARGET_DML_DIALECTS = new Set([
|
|
284787
|
+
"postgres",
|
|
284788
|
+
"snowflake",
|
|
284789
|
+
"standardsql",
|
|
284790
|
+
"duckdb"
|
|
284791
|
+
]);
|
|
283309
284792
|
var MERGE_CAPABLE_DIALECTS = new Set([
|
|
283310
284793
|
"postgres",
|
|
283311
284794
|
"snowflake",
|
|
283312
|
-
"standardsql"
|
|
284795
|
+
"standardsql",
|
|
284796
|
+
"duckdb"
|
|
283313
284797
|
]);
|
|
283314
284798
|
var DIALECT_DISPLAY_NAMES = {
|
|
283315
284799
|
standardsql: "standardsql (BigQuery)"
|
|
@@ -283326,9 +284810,9 @@ function escapeSqlString(value, dialect) {
|
|
|
283326
284810
|
return dialect === "snowflake" || dialect === "standardsql" ? quoted.replace(/\\/g, "\\\\") : quoted;
|
|
283327
284811
|
}
|
|
283328
284812
|
function secondsPrecision(value) {
|
|
283329
|
-
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})
|
|
284813
|
+
const match = value.trim().match(/^(\d{4}-\d{2}-\d{2})[T ](\d{2}:\d{2}:\d{2})(?:\.\d+)?Z?$/);
|
|
283330
284814
|
if (!match) {
|
|
283331
|
-
throw new Error(`not
|
|
284815
|
+
throw new Error(`not a UTC ISO-8601 timestamp: ${JSON.stringify(value)} (expected YYYY-MM-DDTHH:MM:SS, with no zone offset)`);
|
|
283332
284816
|
}
|
|
283333
284817
|
return `${match[1]} ${match[2]}`;
|
|
283334
284818
|
}
|
|
@@ -283407,6 +284891,11 @@ var TABLE_PATH_OPTIONS = {
|
|
|
283407
284891
|
quoteChar: "`",
|
|
283408
284892
|
escapeStyle: "doubled",
|
|
283409
284893
|
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_-]*/
|
|
284894
|
+
},
|
|
284895
|
+
duckdb: {
|
|
284896
|
+
quoteChar: '"',
|
|
284897
|
+
escapeStyle: "doubled",
|
|
284898
|
+
bareIdentRegex: /^[A-Za-z_][A-Za-z0-9_$-]*/
|
|
283410
284899
|
}
|
|
283411
284900
|
};
|
|
283412
284901
|
function decodeTablePathSegments(dialect, tablePath) {
|
|
@@ -283445,7 +284934,17 @@ async function probeTargetColumns(runner, dialect, physicalTableName) {
|
|
|
283445
284934
|
};
|
|
283446
284935
|
}
|
|
283447
284936
|
let sql;
|
|
283448
|
-
if (dialect === "
|
|
284937
|
+
if (dialect === "duckdb") {
|
|
284938
|
+
const table = segments[segments.length - 1];
|
|
284939
|
+
const schema = segments.length >= 3 ? segments[segments.length - 2] : "main";
|
|
284940
|
+
const database = segments[0];
|
|
284941
|
+
sql = `SELECT column_name AS ${probeAlias("column_name", dialect)}
|
|
284942
|
+
FROM duckdb_columns()
|
|
284943
|
+
WHERE database_name = '${escapeSqlString(database, dialect)}'
|
|
284944
|
+
AND schema_name = '${escapeSqlString(schema, dialect)}'
|
|
284945
|
+
AND table_name = '${escapeSqlString(table, dialect)}'
|
|
284946
|
+
ORDER BY column_index`;
|
|
284947
|
+
} else if (dialect === "postgres" || dialect === "snowflake") {
|
|
283449
284948
|
const table = segments[segments.length - 1];
|
|
283450
284949
|
const schema = segments.length >= 2 ? `'${escapeSqlString(segments[segments.length - 2], dialect)}'` : "current_schema()";
|
|
283451
284950
|
const infoSchema = dialect === "snowflake" && segments.length >= 3 ? `${quoteIdentifier(segments[segments.length - 3], dialect)}.information_schema.columns` : "information_schema.columns";
|
|
@@ -283601,11 +285100,12 @@ function deltaScript(statements) {
|
|
|
283601
285100
|
return statements.map((s) => `${s};`).join(`
|
|
283602
285101
|
`);
|
|
283603
285102
|
}
|
|
285103
|
+
var SELF_ROLLING_BACK_DIALECTS = new Set;
|
|
283604
285104
|
async function applyDeltaScript(runner, dialect, statements) {
|
|
283605
285105
|
try {
|
|
283606
285106
|
await runner(deltaScript(statements));
|
|
283607
285107
|
} catch (err) {
|
|
283608
|
-
if (statements.length > 1) {
|
|
285108
|
+
if (statements.length > 1 && !SELF_ROLLING_BACK_DIALECTS.has(dialect)) {
|
|
283609
285109
|
try {
|
|
283610
285110
|
await runner(transactionKeywords(dialect).rollback);
|
|
283611
285111
|
} catch (rollbackErr) {
|
|
@@ -283622,6 +285122,12 @@ function ledgerLineageMismatch(entry, lineage) {
|
|
|
283622
285122
|
reason: `the recorded boundary belongs to table ` + `"${entry.physicalTableName}", not "${lineage.physicalTableName}" — ` + `either this table was renamed between runs, or another source ` + `shares this source's content address and advanced the boundary ` + `against its own table`
|
|
283623
285123
|
};
|
|
283624
285124
|
}
|
|
285125
|
+
if ((entry.storageDestinationName ?? undefined) !== (lineage.storageDestinationName ?? undefined)) {
|
|
285126
|
+
return {
|
|
285127
|
+
reasonCode: "table_renamed",
|
|
285128
|
+
reason: `the recorded boundary belongs to a table in ` + `${entry.storageDestinationName ? `storage destination "${entry.storageDestinationName}"` : "the source warehouse"}` + `, while this refresh writes to ` + `${lineage.storageDestinationName ? `storage destination "${lineage.storageDestinationName}"` : "the source warehouse"}`
|
|
285129
|
+
};
|
|
285130
|
+
}
|
|
283625
285131
|
const changed = (reason) => ({ reasonCode: "lineage_changed", reason });
|
|
283626
285132
|
if (entry.sourceEntityId !== lineage.sourceEntityId) {
|
|
283627
285133
|
return changed(`the source's content address is not the one the recorded boundary was ` + `measured under, so that boundary describes different SQL than this ` + `refresh would apply a delta to — either the model was edited, or ` + `this refresh is instructed through a different version of the ` + `package than the one that advanced the boundary`);
|
|
@@ -283644,15 +285150,33 @@ function ledgerLineageMismatch(entry, lineage) {
|
|
|
283644
285150
|
}
|
|
283645
285151
|
return;
|
|
283646
285152
|
}
|
|
283647
|
-
async function resolveDeltaEnd(
|
|
285153
|
+
async function resolveDeltaEnd(target, lineage, now) {
|
|
283648
285154
|
const type = lineage.watermarkType;
|
|
283649
285155
|
if (type === "date" || type === "timestamp") {
|
|
283650
285156
|
return { bound: snapshotBound(type, now) };
|
|
283651
285157
|
}
|
|
283652
|
-
return
|
|
285158
|
+
return target.probeSourceFrontier();
|
|
285159
|
+
}
|
|
285160
|
+
function warehouseDeltaTarget(params) {
|
|
285161
|
+
const { dialect, runner, lineage, sourceSQL } = params;
|
|
285162
|
+
return {
|
|
285163
|
+
dialect,
|
|
285164
|
+
runner,
|
|
285165
|
+
quotedTablePath: params.quotedTablePath,
|
|
285166
|
+
logicalTablePath: lineage.physicalTableName,
|
|
285167
|
+
deltaRows: async (start, end) => deltaSelect({
|
|
285168
|
+
dialect,
|
|
285169
|
+
sourceSQL,
|
|
285170
|
+
watermarkName: lineage.watermarkName,
|
|
285171
|
+
start,
|
|
285172
|
+
end
|
|
285173
|
+
}),
|
|
285174
|
+
probeSourceFrontier: () => probeMaxWatermark(runner, dialect, sourceSQL, lineage.watermarkName, lineage.watermarkType)
|
|
285175
|
+
};
|
|
283653
285176
|
}
|
|
283654
285177
|
async function planIncrementalStep(inputs) {
|
|
283655
|
-
const {
|
|
285178
|
+
const { target, lineage, ledgerEntry } = inputs;
|
|
285179
|
+
const { runner, dialect } = target;
|
|
283656
285180
|
if (inputs.forceRefresh) {
|
|
283657
285181
|
return {
|
|
283658
285182
|
mode: "seed",
|
|
@@ -283678,7 +285202,7 @@ async function planIncrementalStep(inputs) {
|
|
|
283678
285202
|
reason: `merge_key= needs MERGE, which this Postgres server ` + `(server_version_num ${inputs.postgresVersionNum ?? "unknown"}) ` + `does not have; MERGE requires Postgres 15`
|
|
283679
285203
|
};
|
|
283680
285204
|
}
|
|
283681
|
-
const probe = await probeTargetColumns(runner, dialect,
|
|
285205
|
+
const probe = await probeTargetColumns(runner, dialect, target.logicalTablePath);
|
|
283682
285206
|
if (probe.columns.length === 0) {
|
|
283683
285207
|
return {
|
|
283684
285208
|
mode: "seed",
|
|
@@ -283686,7 +285210,7 @@ async function planIncrementalStep(inputs) {
|
|
|
283686
285210
|
reason: `the target table could not be described` + (probe.error ? ` (${probe.error})` : ", so it is treated as absent")
|
|
283687
285211
|
};
|
|
283688
285212
|
}
|
|
283689
|
-
const rows = await probeTargetNonEmpty(runner, dialect,
|
|
285213
|
+
const rows = await probeTargetNonEmpty(runner, dialect, target.quotedTablePath);
|
|
283690
285214
|
if (!rows.nonEmpty) {
|
|
283691
285215
|
return {
|
|
283692
285216
|
mode: "seed",
|
|
@@ -283698,7 +285222,7 @@ async function planIncrementalStep(inputs) {
|
|
|
283698
285222
|
malloyType: ledgerEntry.coveredThroughType,
|
|
283699
285223
|
value: ledgerEntry.coveredThroughValue
|
|
283700
285224
|
};
|
|
283701
|
-
const end = await resolveDeltaEnd(
|
|
285225
|
+
const end = await resolveDeltaEnd(target, lineage, inputs.now);
|
|
283702
285226
|
if (!end.bound) {
|
|
283703
285227
|
return {
|
|
283704
285228
|
mode: "skip",
|
|
@@ -283725,14 +285249,8 @@ async function planIncrementalStep(inputs) {
|
|
|
283725
285249
|
}
|
|
283726
285250
|
const statements = deltaStatements({
|
|
283727
285251
|
dialect,
|
|
283728
|
-
quotedTablePath:
|
|
283729
|
-
deltaSQL:
|
|
283730
|
-
dialect,
|
|
283731
|
-
sourceSQL: inputs.sourceSQL,
|
|
283732
|
-
watermarkName: lineage.watermarkName,
|
|
283733
|
-
start,
|
|
283734
|
-
end: end.bound
|
|
283735
|
-
}),
|
|
285252
|
+
quotedTablePath: target.quotedTablePath,
|
|
285253
|
+
deltaSQL: await target.deltaRows(start, end.bound),
|
|
283736
285254
|
columns: inputs.columns,
|
|
283737
285255
|
mergeKeys: lineage.mergeKeys,
|
|
283738
285256
|
watermarkName: lineage.watermarkName,
|
|
@@ -283814,14 +285332,11 @@ function rejectionsForSource(source) {
|
|
|
283814
285332
|
if (!d.incremental)
|
|
283815
285333
|
return out;
|
|
283816
285334
|
const dialect = source.dialect ?? "";
|
|
283817
|
-
if (!
|
|
283818
|
-
out.push(`${where} declares ${MODE}, which is not supported on dialect ` + `"${dialect || "unknown"}". Incremental refresh
|
|
285335
|
+
if (!INCREMENTAL_SOURCE_RANGE_DIALECTS.has(dialect)) {
|
|
285336
|
+
out.push(`${where} declares ${MODE}, which is not supported on dialect ` + `"${dialect || "unknown"}". Incremental refresh reads a bounded ` + `watermark range out of the source and applies it transactionally, ` + `so it is enabled only where the publisher has proven both: ` + `${describeDialects(INCREMENTAL_SOURCE_RANGE_DIALECTS)}. Use ` + `refresh="full" here; the supported set widens in a later release.`);
|
|
283819
285337
|
} else if (d.declaredMergeKey && !MERGE_CAPABLE_DIALECTS.has(dialect)) {
|
|
283820
285338
|
out.push(`${where} declares merge_key=, which needs a MERGE statement that ` + `dialect "${dialect}" does not have. Remove merge_key= to replace ` + `the watermark range instead of merging by row identity.`);
|
|
283821
285339
|
}
|
|
283822
|
-
if (source.storageDestination) {
|
|
283823
|
-
out.push(`${where} declares ${MODE} together with ` + `storage="${source.storageDestination}". A stored source is ` + `materialized into the storage destination's own engine, while an ` + `incremental delta is applied on the source warehouse — so the two ` + `cannot be combined yet. Drop one: refresh="full" keeps the storage ` + `destination, removing storage= keeps the incremental refresh in the ` + `source warehouse.`);
|
|
283824
|
-
}
|
|
283825
285340
|
const dangling = unresolved([d.watermark, ...d.mergeKeys]);
|
|
283826
285341
|
if (dangling.length > 0) {
|
|
283827
285342
|
out.push(`${where} declares ${quoteList(dangling.map((n) => n.name))} which ` + `${dangling.length === 1 ? "is not" : "are not"} ` + `${dangling.length === 1 ? "a column" : "columns"} of the source's ` + `materialized output. Available: ${quoteList(d.outputColumns)}. Note ` + `that a derived dimension: on a table-backed source is queryable but ` + `NOT materialized, so it cannot be a watermark or merge key.`);
|
|
@@ -283892,9 +285407,10 @@ function incrementalPolicyAdvisories(sources) {
|
|
|
283892
285407
|
});
|
|
283893
285408
|
}
|
|
283894
285409
|
if (d.incremental && d.watermark !== undefined && !d.declaredMergeKey) {
|
|
285410
|
+
const rangeDeleteCost = source.storageDestination ? `(b) the range DELETE rewrites every stored file the range ` + `touches, so a wide range costs more than its row count suggests — ` + `a narrower refresh interval keeps each one small.` : `(b) if the destination table is not partitioned or clustered on ` + `"${d.watermark.name}", the range DELETE may re-read the whole ` + `table on every run.`;
|
|
283895
285411
|
warnings.push({
|
|
283896
285412
|
...at,
|
|
283897
|
-
message: `is ${MODE} with no merge_key=, so each refresh REPLACES the ` + `watermark range it just computed. Two consequences worth ` + `checking: (a) a row that is later restated with an ADVANCED ` + `watermark value appears twice rather than replacing its ` + `predecessor, since the older row sits below the new range — ` + `declare merge_key= with the row's identity dimensions if that ` + `happens;
|
|
285413
|
+
message: `is ${MODE} with no merge_key=, so each refresh REPLACES the ` + `watermark range it just computed. Two consequences worth ` + `checking: (a) a row that is later restated with an ADVANCED ` + `watermark value appears twice rather than replacing its ` + `predecessor, since the older row sits below the new range — ` + `declare merge_key= with the row's identity dimensions if that ` + `happens; ${rangeDeleteCost}`
|
|
283898
285414
|
});
|
|
283899
285415
|
}
|
|
283900
285416
|
}
|
|
@@ -287302,7 +288818,11 @@ class EnvironmentStore {
|
|
|
287302
288818
|
assertSafeEnvironmentPath(absoluteEnvironmentPath);
|
|
287303
288819
|
const startedAt = Date.now();
|
|
287304
288820
|
logger.info(`Detected zip file at "${absoluteEnvironmentPath}". Unzipping...`);
|
|
287305
|
-
const
|
|
288821
|
+
const archiveDir = path11.resolve(path11.dirname(absoluteEnvironmentPath));
|
|
288822
|
+
const unzippedEnvironmentPath = path11.resolve(archiveDir, path11.basename(absoluteEnvironmentPath, ".zip"));
|
|
288823
|
+
if (!unzippedEnvironmentPath.startsWith(archiveDir + path11.sep)) {
|
|
288824
|
+
throw new BadRequestError(`Refusing to unzip "${absoluteEnvironmentPath}": target escapes its directory`);
|
|
288825
|
+
}
|
|
287306
288826
|
await fs9.promises.rm(unzippedEnvironmentPath, {
|
|
287307
288827
|
recursive: true,
|
|
287308
288828
|
force: true
|
|
@@ -288288,6 +289808,9 @@ class MaterializationController {
|
|
|
288288
289808
|
if (entry.mergeKeys !== undefined && (!Array.isArray(entry.mergeKeys) || entry.mergeKeys.some((k) => typeof k !== "string"))) {
|
|
288289
289809
|
throw new BadRequestError("Ledger entry 'mergeKeys' must be an array of strings");
|
|
288290
289810
|
}
|
|
289811
|
+
if (entry.storageDestinationName !== undefined && (typeof entry.storageDestinationName !== "string" || entry.storageDestinationName === "")) {
|
|
289812
|
+
throw new BadRequestError("Ledger entry 'storageDestinationName' must be a non-empty string when present");
|
|
289813
|
+
}
|
|
288291
289814
|
return {
|
|
288292
289815
|
connectionName: entry.connectionName,
|
|
288293
289816
|
physicalTableName: entry.physicalTableName,
|
|
@@ -288296,7 +289819,8 @@ class MaterializationController {
|
|
|
288296
289819
|
watermark: entry.watermark,
|
|
288297
289820
|
strategy: entry.strategy,
|
|
288298
289821
|
sourceEntityId: entry.sourceEntityId,
|
|
288299
|
-
...Array.isArray(entry.mergeKeys) ? { mergeKeys: entry.mergeKeys } : {}
|
|
289822
|
+
...Array.isArray(entry.mergeKeys) ? { mergeKeys: entry.mergeKeys } : {},
|
|
289823
|
+
...typeof entry.storageDestinationName === "string" ? { storageDestinationName: entry.storageDestinationName } : {}
|
|
288300
289824
|
};
|
|
288301
289825
|
}
|
|
288302
289826
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
@@ -296967,13 +298491,21 @@ init_build_plan();
|
|
|
296967
298491
|
init_logger();
|
|
296968
298492
|
init_materialization_metrics();
|
|
296969
298493
|
init_utils4();
|
|
296970
|
-
function ledgerTableKey(
|
|
296971
|
-
return
|
|
298494
|
+
function ledgerTableKey(table) {
|
|
298495
|
+
return [
|
|
298496
|
+
table.connectionName,
|
|
298497
|
+
table.storageDestinationName ?? "",
|
|
298498
|
+
table.physicalTableName
|
|
298499
|
+
].join("\x00");
|
|
296972
298500
|
}
|
|
296973
298501
|
function indexCallerLedger(entries) {
|
|
296974
298502
|
const index = new Map;
|
|
296975
298503
|
for (const entry of entries) {
|
|
296976
|
-
index.set(ledgerTableKey(
|
|
298504
|
+
index.set(ledgerTableKey({
|
|
298505
|
+
connectionName: entry.connectionName,
|
|
298506
|
+
storageDestinationName: entry.storageDestinationName,
|
|
298507
|
+
physicalTableName: entry.physicalTableName
|
|
298508
|
+
}), {
|
|
296977
298509
|
sourceEntityId: entry.sourceEntityId,
|
|
296978
298510
|
coveredThroughValue: entry.coveredThrough,
|
|
296979
298511
|
coveredThroughType: entry.coveredThroughType,
|
|
@@ -296981,7 +298513,8 @@ function indexCallerLedger(entries) {
|
|
|
296981
298513
|
mergeKeyDimensions: entry.mergeKeys ?? [],
|
|
296982
298514
|
derivedStrategy: entry.strategy,
|
|
296983
298515
|
physicalTableName: entry.physicalTableName,
|
|
296984
|
-
connectionName: entry.connectionName
|
|
298516
|
+
connectionName: entry.connectionName,
|
|
298517
|
+
storageDestinationName: entry.storageDestinationName
|
|
296985
298518
|
});
|
|
296986
298519
|
}
|
|
296987
298520
|
return index;
|
|
@@ -296994,16 +298527,18 @@ function incrementalLineage(params) {
|
|
|
296994
298527
|
if (watermark === undefined || watermark.kind !== "dimension" || !d.watermarkOrderable || !isRenderableWatermarkType(watermark.malloyType) || d.strategy === undefined) {
|
|
296995
298528
|
return;
|
|
296996
298529
|
}
|
|
296997
|
-
if (!
|
|
298530
|
+
if (!INCREMENTAL_SOURCE_RANGE_DIALECTS.has(params.dialect))
|
|
296998
298531
|
return;
|
|
296999
|
-
if (params.
|
|
298532
|
+
if (!INCREMENTAL_TARGET_DML_DIALECTS.has(params.targetDialect)) {
|
|
297000
298533
|
return;
|
|
298534
|
+
}
|
|
297001
298535
|
const mergeKeys = d.mergeKeys.filter((k) => k.kind === "dimension").map((k) => k.name);
|
|
297002
298536
|
if (mergeKeys.length !== d.mergeKeys.length)
|
|
297003
298537
|
return;
|
|
297004
298538
|
return {
|
|
297005
298539
|
physicalTableName: params.physicalTableName,
|
|
297006
298540
|
connectionName: params.connectionName,
|
|
298541
|
+
storageDestinationName: params.storageDestinationName,
|
|
297007
298542
|
sourceEntityId: params.sourceEntityId,
|
|
297008
298543
|
watermarkName: watermark.name,
|
|
297009
298544
|
watermarkType: watermark.malloyType,
|
|
@@ -297012,15 +298547,14 @@ function incrementalLineage(params) {
|
|
|
297012
298547
|
};
|
|
297013
298548
|
}
|
|
297014
298549
|
async function planSourceRefresh(params) {
|
|
297015
|
-
const { context, lineage } = params;
|
|
298550
|
+
const { context, lineage, target } = params;
|
|
297016
298551
|
const forceRefresh = context.forceRefresh || params.reseed === true;
|
|
297017
|
-
const dialect = params.persistSource.dialectName;
|
|
297018
298552
|
let ledgerEntry = null;
|
|
297019
298553
|
if (context.callerLedger) {
|
|
297020
|
-
ledgerEntry = context.callerLedger.get(ledgerTableKey(lineage
|
|
298554
|
+
ledgerEntry = context.callerLedger.get(ledgerTableKey(lineage)) ?? null;
|
|
297021
298555
|
} else {
|
|
297022
298556
|
try {
|
|
297023
|
-
ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, lineage
|
|
298557
|
+
ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, lineage);
|
|
297024
298558
|
} catch (err) {
|
|
297025
298559
|
return {
|
|
297026
298560
|
mode: "seed",
|
|
@@ -297029,18 +298563,15 @@ async function planSourceRefresh(params) {
|
|
|
297029
298563
|
};
|
|
297030
298564
|
}
|
|
297031
298565
|
}
|
|
297032
|
-
const postgresVersionNum = lineage.strategy === "merge" && dialect === "postgres" && ledgerEntry !== null ? await probePostgresVersion(
|
|
298566
|
+
const postgresVersionNum = lineage.strategy === "merge" && target.dialect === "postgres" && ledgerEntry !== null ? await probePostgresVersion(target.runner) : undefined;
|
|
297033
298567
|
let step;
|
|
297034
298568
|
try {
|
|
297035
298569
|
step = await planIncrementalStep({
|
|
297036
|
-
|
|
297037
|
-
dialect,
|
|
297038
|
-
quotedTablePath: params.quotedTablePath,
|
|
298570
|
+
target,
|
|
297039
298571
|
lineage,
|
|
297040
298572
|
ledgerEntry,
|
|
297041
298573
|
forceRefresh,
|
|
297042
298574
|
now: context.now,
|
|
297043
|
-
sourceSQL: params.sourceSQL,
|
|
297044
298575
|
columns: params.columns,
|
|
297045
298576
|
postgresVersionNum
|
|
297046
298577
|
});
|
|
@@ -297072,6 +298603,7 @@ async function advanceLedger(params) {
|
|
|
297072
298603
|
derivedStrategy: lineage.strategy,
|
|
297073
298604
|
physicalTableName: lineage.physicalTableName,
|
|
297074
298605
|
connectionName: lineage.connectionName,
|
|
298606
|
+
storageDestinationName: lineage.storageDestinationName,
|
|
297075
298607
|
advancedByMaterializationId: context.materializationId
|
|
297076
298608
|
});
|
|
297077
298609
|
} catch (err) {
|
|
@@ -297087,7 +298619,7 @@ async function resetLedger(context, lineage) {
|
|
|
297087
298619
|
if (context.callerLedger)
|
|
297088
298620
|
return;
|
|
297089
298621
|
try {
|
|
297090
|
-
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, lineage
|
|
298622
|
+
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, lineage);
|
|
297091
298623
|
} catch (err) {
|
|
297092
298624
|
logger.warn("Failed to clear the covered_through boundary", {
|
|
297093
298625
|
packageName: context.packageName,
|
|
@@ -297115,6 +298647,48 @@ async function advanceLedgerAfterSeed(params) {
|
|
|
297115
298647
|
});
|
|
297116
298648
|
return boundary.bound;
|
|
297117
298649
|
}
|
|
298650
|
+
async function applyIncrementalStep(params) {
|
|
298651
|
+
const { context, lineage, step, target } = params;
|
|
298652
|
+
if (step.mode !== "delta") {
|
|
298653
|
+
reportIncrementalStep({
|
|
298654
|
+
step,
|
|
298655
|
+
sourceName: params.sourceName,
|
|
298656
|
+
packageName: context.packageName,
|
|
298657
|
+
physicalTableName: lineage.physicalTableName
|
|
298658
|
+
});
|
|
298659
|
+
}
|
|
298660
|
+
if (step.mode === "seed")
|
|
298661
|
+
return { applied: false };
|
|
298662
|
+
if (step.mode === "skip") {
|
|
298663
|
+
return {
|
|
298664
|
+
applied: true,
|
|
298665
|
+
coveredThrough: step.coveredThrough,
|
|
298666
|
+
refresh: "none"
|
|
298667
|
+
};
|
|
298668
|
+
}
|
|
298669
|
+
const startTime = performance.now();
|
|
298670
|
+
await applyDeltaScript(target.runner, target.dialect, step.statements);
|
|
298671
|
+
await advanceLedger({
|
|
298672
|
+
context,
|
|
298673
|
+
lineage,
|
|
298674
|
+
coveredThrough: step.coveredThrough
|
|
298675
|
+
});
|
|
298676
|
+
const durationMs = Math.round(performance.now() - startTime);
|
|
298677
|
+
reportDeltaApplied({
|
|
298678
|
+
packageName: context.packageName,
|
|
298679
|
+
sourceName: params.sourceName,
|
|
298680
|
+
physicalTableName: lineage.physicalTableName,
|
|
298681
|
+
rangeStart: step.start.value,
|
|
298682
|
+
rangeEnd: step.end.value,
|
|
298683
|
+
durationMs
|
|
298684
|
+
});
|
|
298685
|
+
return {
|
|
298686
|
+
applied: true,
|
|
298687
|
+
durationMs,
|
|
298688
|
+
coveredThrough: step.coveredThrough,
|
|
298689
|
+
refresh: "delta"
|
|
298690
|
+
};
|
|
298691
|
+
}
|
|
297118
298692
|
function reportIncrementalStep(params) {
|
|
297119
298693
|
const { step, sourceName, packageName, physicalTableName } = params;
|
|
297120
298694
|
recordIncrementalStep(step.mode, step.reasonCode);
|
|
@@ -297286,6 +298860,7 @@ var PASSTHROUGH_SOURCE_TYPES = [
|
|
|
297286
298860
|
"postgres"
|
|
297287
298861
|
];
|
|
297288
298862
|
var STORAGE_DESTINATION_TYPES = ["ducklake", "duckdb"];
|
|
298863
|
+
var STORAGE_TARGET_DIALECT = "duckdb";
|
|
297289
298864
|
function wrapPassthrough(sourceType, handle, innerSQL) {
|
|
297290
298865
|
const sql = `'${escapeSQL(innerSQL)}'`;
|
|
297291
298866
|
const h = `'${escapeSQL(handle)}'`;
|
|
@@ -297464,7 +299039,9 @@ function createIsolatedBuildSession(sessionName) {
|
|
|
297464
299039
|
const workDir = mkdtempSync(path14.join(os4.tmpdir(), "malloy-build-"));
|
|
297465
299040
|
let session;
|
|
297466
299041
|
try {
|
|
297467
|
-
session = new DuckDBConnection4(sessionName, ":memory:", workDir
|
|
299042
|
+
session = new DuckDBConnection4(sessionName, ":memory:", workDir, {
|
|
299043
|
+
rowLimit: Number.MAX_SAFE_INTEGER
|
|
299044
|
+
});
|
|
297468
299045
|
} catch (err) {
|
|
297469
299046
|
rmSync(workDir, { recursive: true, force: true });
|
|
297470
299047
|
throw err;
|
|
@@ -297499,6 +299076,9 @@ function createIsolatedBuildSession(sessionName) {
|
|
|
297499
299076
|
};
|
|
297500
299077
|
return { session, dispose };
|
|
297501
299078
|
}
|
|
299079
|
+
async function pinSessionToUTC(session) {
|
|
299080
|
+
await session.runSQL("SET TimeZone='UTC'");
|
|
299081
|
+
}
|
|
297502
299082
|
async function buildSourceIntoStorage(params) {
|
|
297503
299083
|
const {
|
|
297504
299084
|
destinationName,
|
|
@@ -297518,12 +299098,34 @@ async function buildSourceIntoStorage(params) {
|
|
|
297518
299098
|
if (destinationConnection.type === "ducklake") {
|
|
297519
299099
|
await session.runSQL("SET ducklake_default_data_inlining_row_limit=0");
|
|
297520
299100
|
}
|
|
299101
|
+
await pinSessionToUTC(session);
|
|
297521
299102
|
const federated = await federateSourceForPassthrough(session, sourceType, sourceFederationConfig(sourceConnection));
|
|
297522
299103
|
await tagSnowflakeSession(session, sourceType, federated.handle, queryMetadata);
|
|
297523
|
-
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`,
|
|
299104
|
+
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, STORAGE_TARGET_DIALECT);
|
|
299105
|
+
if (params.incremental) {
|
|
299106
|
+
const refreshed = await params.incremental.plan({
|
|
299107
|
+
session,
|
|
299108
|
+
sourceType,
|
|
299109
|
+
handle: federated.handle,
|
|
299110
|
+
quotedTablePath: target
|
|
299111
|
+
});
|
|
299112
|
+
if (refreshed) {
|
|
299113
|
+
return {
|
|
299114
|
+
storageDestinationName: destinationName,
|
|
299115
|
+
schema: await describeTable(session, target),
|
|
299116
|
+
readCost: refreshed.readCost,
|
|
299117
|
+
refresh: refreshed
|
|
299118
|
+
};
|
|
299119
|
+
}
|
|
299120
|
+
}
|
|
297524
299121
|
const read = await issuePassthroughRead(session, sourceType, federated.handle, buildSQL, queryMetadata);
|
|
297525
299122
|
const schema = await createTableAndDescribe(session, target, read.selectSQL);
|
|
299123
|
+
const seededThrough = await params.incremental?.afterSeed({
|
|
299124
|
+
session,
|
|
299125
|
+
quotedTablePath: target
|
|
299126
|
+
});
|
|
297526
299127
|
return {
|
|
299128
|
+
seededThrough,
|
|
297527
299129
|
storageDestinationName: destinationName,
|
|
297528
299130
|
schema,
|
|
297529
299131
|
readCost: read.cost ?? await snowflakeReadCostAfterBuild(session, sourceType, federated.handle, buildSQL, queryMetadata, sourceConnection.snowflakeConnection?.database)
|
|
@@ -297550,6 +299152,7 @@ async function buildDownstreamIntoStorage(params) {
|
|
|
297550
299152
|
if (destinationConnection.type === "ducklake") {
|
|
297551
299153
|
await session.runSQL("SET ducklake_default_data_inlining_row_limit=0");
|
|
297552
299154
|
}
|
|
299155
|
+
await pinSessionToUTC(session);
|
|
297553
299156
|
const root = "file:///chained-build/";
|
|
297554
299157
|
const url2 = `${root}m.malloy`;
|
|
297555
299158
|
const runtime = new Runtime4({
|
|
@@ -297578,7 +299181,7 @@ async function buildDownstreamIntoStorage(params) {
|
|
|
297578
299181
|
});
|
|
297579
299182
|
}
|
|
297580
299183
|
const sql = projectToPublicColumns(downstream, downstream.getSQL({ virtualMap }));
|
|
297581
|
-
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`,
|
|
299184
|
+
const target = quoteManifestTablePath(`${destinationName}.${physicalTableName}`, STORAGE_TARGET_DIALECT);
|
|
297582
299185
|
const schema = await createTableAndDescribe(session, target, sql);
|
|
297583
299186
|
return {
|
|
297584
299187
|
storageDestinationName: destinationName,
|
|
@@ -297602,7 +299205,7 @@ async function assertStorageServeShapeCompiles(params) {
|
|
|
297602
299205
|
await assertServesInDuckDB(sourceName, binding, new FixedConnectionMap3(new Map([[destinationName, sharedGateSession]]), destinationName));
|
|
297603
299206
|
}
|
|
297604
299207
|
function dropStorageTableSql(destinationName, physicalTableName) {
|
|
297605
|
-
return `DROP TABLE IF EXISTS ${quoteManifestTablePath(`${destinationName}.${physicalTableName}`,
|
|
299208
|
+
return `DROP TABLE IF EXISTS ${quoteManifestTablePath(`${destinationName}.${physicalTableName}`, STORAGE_TARGET_DIALECT)}`;
|
|
297606
299209
|
}
|
|
297607
299210
|
async function dropStorageTable(params) {
|
|
297608
299211
|
const {
|
|
@@ -297632,7 +299235,7 @@ async function attachDestinationReadWrite(session, destinationName, destinationC
|
|
|
297632
299235
|
const destinationRoot = storageDestinationRoot(environmentPath);
|
|
297633
299236
|
mkdirSync2(destinationRoot, { recursive: true });
|
|
297634
299237
|
const dbPath = path14.join(destinationRoot, `${destinationName}.duckdb`);
|
|
297635
|
-
await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName,
|
|
299238
|
+
await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName, STORAGE_TARGET_DIALECT)}`);
|
|
297636
299239
|
}
|
|
297637
299240
|
function assertSupportedDestination(destinationName, destinationConnection) {
|
|
297638
299241
|
const type = destinationConnection.type ?? "";
|
|
@@ -297678,6 +299281,65 @@ async function describeTable(session, quotedTablePath) {
|
|
|
297678
299281
|
return columns;
|
|
297679
299282
|
}
|
|
297680
299283
|
|
|
299284
|
+
// src/service/incremental_storage.ts
|
|
299285
|
+
init_build_plan();
|
|
299286
|
+
init_quoting();
|
|
299287
|
+
init_utils4();
|
|
299288
|
+
function storageDeltaTarget(params) {
|
|
299289
|
+
const {
|
|
299290
|
+
session,
|
|
299291
|
+
sourceType,
|
|
299292
|
+
handle,
|
|
299293
|
+
destinationName,
|
|
299294
|
+
lineage,
|
|
299295
|
+
persistSource,
|
|
299296
|
+
buildSQL,
|
|
299297
|
+
queryMetadata,
|
|
299298
|
+
quotedTablePath
|
|
299299
|
+
} = params;
|
|
299300
|
+
const sourceDialect = persistSource.dialectName;
|
|
299301
|
+
const logicalTablePath = `${destinationName}.${params.physicalTableName}`;
|
|
299302
|
+
let readCost = null;
|
|
299303
|
+
return {
|
|
299304
|
+
readCost: () => readCost,
|
|
299305
|
+
target: {
|
|
299306
|
+
dialect: STORAGE_TARGET_DIALECT,
|
|
299307
|
+
runner: (sql) => session.runSQL(sql),
|
|
299308
|
+
quotedTablePath,
|
|
299309
|
+
logicalTablePath,
|
|
299310
|
+
deltaRows: async (start, end) => {
|
|
299311
|
+
const bounded = projectToPublicColumns(persistSource, deltaSelect({
|
|
299312
|
+
dialect: sourceDialect,
|
|
299313
|
+
sourceSQL: buildSQL,
|
|
299314
|
+
watermarkName: lineage.watermarkName,
|
|
299315
|
+
start,
|
|
299316
|
+
end
|
|
299317
|
+
}));
|
|
299318
|
+
const read = await issuePassthroughRead(session, sourceType, handle, bounded, queryMetadata);
|
|
299319
|
+
readCost = read.cost;
|
|
299320
|
+
return read.selectSQL;
|
|
299321
|
+
},
|
|
299322
|
+
probeSourceFrontier: async () => {
|
|
299323
|
+
const column = quoteIdentifier(lineage.watermarkName, sourceDialect);
|
|
299324
|
+
const alias = quoteIdentifier("watermark_max", sourceDialect);
|
|
299325
|
+
const inner = `SELECT MAX(${column}) AS ${alias} ` + `FROM (${buildSQL}) AS __w`;
|
|
299326
|
+
try {
|
|
299327
|
+
const result = await session.runSQL(wrapPassthrough(sourceType, handle, inner));
|
|
299328
|
+
const rows = result?.rows ?? [];
|
|
299329
|
+
const raw = rows[0]?.["watermark_max"];
|
|
299330
|
+
if (raw === null || raw === undefined)
|
|
299331
|
+
return {};
|
|
299332
|
+
return {
|
|
299333
|
+
bound: canonicalBoundValue(lineage.watermarkType, raw)
|
|
299334
|
+
};
|
|
299335
|
+
} catch (err) {
|
|
299336
|
+
return { error: errMessage(err) };
|
|
299337
|
+
}
|
|
299338
|
+
}
|
|
299339
|
+
}
|
|
299340
|
+
};
|
|
299341
|
+
}
|
|
299342
|
+
|
|
297681
299343
|
// src/service/materialization_service.ts
|
|
297682
299344
|
init_connection();
|
|
297683
299345
|
init_materialization_serve_transform();
|
|
@@ -297706,6 +299368,7 @@ function ledgerFields(lineage, bound) {
|
|
|
297706
299368
|
ledger: {
|
|
297707
299369
|
connectionName: lineage.connectionName,
|
|
297708
299370
|
physicalTableName: lineage.physicalTableName,
|
|
299371
|
+
...lineage.storageDestinationName ? { storageDestinationName: lineage.storageDestinationName } : {},
|
|
297709
299372
|
coveredThrough: bound.value,
|
|
297710
299373
|
coveredThroughType: bound.malloyType,
|
|
297711
299374
|
watermark: lineage.watermarkName,
|
|
@@ -297804,6 +299467,28 @@ var VALID_TRANSITIONS = {
|
|
|
297804
299467
|
FAILED: [],
|
|
297805
299468
|
CANCELLED: []
|
|
297806
299469
|
};
|
|
299470
|
+
function tableKeyOf(connectionName, storageDestinationName, physicalTableName) {
|
|
299471
|
+
return [
|
|
299472
|
+
connectionName,
|
|
299473
|
+
storageDestinationName ?? "",
|
|
299474
|
+
physicalTableName
|
|
299475
|
+
].join("\x00");
|
|
299476
|
+
}
|
|
299477
|
+
function reportRetainedStorageTables(retained, packageName) {
|
|
299478
|
+
for (const entry of retained) {
|
|
299479
|
+
recordStorageTableRetained(entry.storageDestinationName ?? "unknown");
|
|
299480
|
+
logger.warn("A failed run left a table in a storage destination and did not reclaim " + "it: the source is refreshed incrementally, so this name may be the " + "one it serves from. Unreferenced if the run was building a fresh " + "generation, in which case reclaiming it needs the destination sweep.", {
|
|
299481
|
+
packageName,
|
|
299482
|
+
sourceName: entry.sourceName,
|
|
299483
|
+
destinationName: entry.storageDestinationName,
|
|
299484
|
+
physicalTableName: entry.physicalTableName,
|
|
299485
|
+
refresh: entry.refresh
|
|
299486
|
+
});
|
|
299487
|
+
}
|
|
299488
|
+
}
|
|
299489
|
+
function isReclaimableStorageTable(entry) {
|
|
299490
|
+
return !!entry.storageDestinationName && entry.refresh === undefined;
|
|
299491
|
+
}
|
|
297807
299492
|
|
|
297808
299493
|
class MaterializationService {
|
|
297809
299494
|
environmentStore;
|
|
@@ -298018,10 +299703,11 @@ class MaterializationService {
|
|
|
298018
299703
|
const deltaEligible = incremental !== undefined && incrementalLineage({
|
|
298019
299704
|
declaration: incremental.declarations[persistSource.sourceID],
|
|
298020
299705
|
dialect: persistSource.dialectName,
|
|
299706
|
+
targetDialect: destination !== undefined ? STORAGE_TARGET_DIALECT : persistSource.dialectName,
|
|
298021
299707
|
physicalTableName: logicalName,
|
|
298022
299708
|
connectionName: persistSource.connectionName,
|
|
298023
|
-
|
|
298024
|
-
|
|
299709
|
+
storageDestinationName: destination,
|
|
299710
|
+
sourceEntityId
|
|
298025
299711
|
}) !== undefined;
|
|
298026
299712
|
const prior = priorEntries[sourceEntityId];
|
|
298027
299713
|
if (!deltaEligible && prior && prior.physicalTableName && (prior.storageDestinationName ?? undefined) === destination) {
|
|
@@ -298174,27 +299860,35 @@ class MaterializationService {
|
|
|
298174
299860
|
if (ledger === undefined || ledger.length === 0)
|
|
298175
299861
|
return;
|
|
298176
299862
|
const instructed = new Map;
|
|
299863
|
+
const instructedIntoStorage = new Set;
|
|
298177
299864
|
const byAddress = new Map(Object.values(plan.sources).map((s) => [s.sourceEntityId, s]));
|
|
298178
299865
|
for (const instruction of instructions) {
|
|
298179
299866
|
const planSource = (instruction.sourceID ? plan.sources[instruction.sourceID] : undefined) ?? byAddress.get(instruction.sourceEntityId);
|
|
298180
299867
|
if (!planSource)
|
|
298181
299868
|
continue;
|
|
298182
|
-
instructed.set(
|
|
299869
|
+
instructed.set(tableKeyOf(planSource.connectionName, instruction.destination, instruction.physicalTableName), {
|
|
298183
299870
|
sourceEntityId: planSource.sourceEntityId,
|
|
298184
299871
|
refresh: planSource.refresh ?? null,
|
|
298185
299872
|
reseed: runReseed || instruction.reseed === true
|
|
298186
299873
|
});
|
|
299874
|
+
if (instruction.destination) {
|
|
299875
|
+
instructedIntoStorage.add(tableKeyOf(planSource.connectionName, undefined, instruction.physicalTableName));
|
|
299876
|
+
}
|
|
298187
299877
|
}
|
|
298188
299878
|
const seen = new Set;
|
|
298189
299879
|
for (const entry of ledger) {
|
|
298190
|
-
const key =
|
|
298191
|
-
const table = `'${entry.physicalTableName}' on connection '${entry.connectionName}'
|
|
299880
|
+
const key = tableKeyOf(entry.connectionName, entry.storageDestinationName, entry.physicalTableName);
|
|
299881
|
+
const table = `'${entry.physicalTableName}' ` + (entry.storageDestinationName ? `in storage destination '${entry.storageDestinationName}'` : `on connection '${entry.connectionName}'`);
|
|
298192
299882
|
if (seen.has(key)) {
|
|
298193
299883
|
throw new BadRequestError(`Ledger entry for table ${table} appears more than once`);
|
|
298194
299884
|
}
|
|
298195
299885
|
seen.add(key);
|
|
298196
299886
|
const target = instructed.get(key);
|
|
298197
299887
|
if (!target) {
|
|
299888
|
+
const storedHere = instructedIntoStorage.has(tableKeyOf(entry.connectionName, undefined, entry.physicalTableName));
|
|
299889
|
+
if (entry.storageDestinationName === undefined && storedHere) {
|
|
299890
|
+
continue;
|
|
299891
|
+
}
|
|
298198
299892
|
throw new BadRequestError(`Ledger entry names table ${table}, which this run's ` + `instructions do not build`);
|
|
298199
299893
|
}
|
|
298200
299894
|
if (target.reseed)
|
|
@@ -298256,6 +299950,7 @@ class MaterializationService {
|
|
|
298256
299950
|
entries[sourceEntityId] = entry;
|
|
298257
299951
|
}
|
|
298258
299952
|
const builtThisRun = [];
|
|
299953
|
+
const retainedThisRun = [];
|
|
298259
299954
|
const failures = {};
|
|
298260
299955
|
const failedReasons = [];
|
|
298261
299956
|
const builtSources = [];
|
|
@@ -298314,8 +300009,11 @@ class MaterializationService {
|
|
|
298314
300009
|
}
|
|
298315
300010
|
builtSources.push(persistSource.name);
|
|
298316
300011
|
entries[sourceEntityId] = entry;
|
|
298317
|
-
if (entry
|
|
300012
|
+
if (isReclaimableStorageTable(entry)) {
|
|
298318
300013
|
builtThisRun.push(entry);
|
|
300014
|
+
} else if (entry.storageDestinationName) {
|
|
300015
|
+
retainedThisRun.push(entry);
|
|
300016
|
+
}
|
|
298319
300017
|
}
|
|
298320
300018
|
}
|
|
298321
300019
|
if (builtSources.length === 0 && failedReasons.length > 0) {
|
|
@@ -298323,6 +300021,7 @@ class MaterializationService {
|
|
|
298323
300021
|
}
|
|
298324
300022
|
} catch (err) {
|
|
298325
300023
|
if (owner) {
|
|
300024
|
+
reportRetainedStorageTables(retainedThisRun, owner.packageName);
|
|
298326
300025
|
await this.reclaimStorageTablesFromFailedRun(builtThisRun, environment, owner);
|
|
298327
300026
|
}
|
|
298328
300027
|
throw err;
|
|
@@ -298442,17 +300141,29 @@ class MaterializationService {
|
|
|
298442
300141
|
connectionDigests
|
|
298443
300142
|
}) !== buildSQL;
|
|
298444
300143
|
const publicBuildSQL = projectToPublicColumns(persistSource, buildSQL);
|
|
298445
|
-
return this.buildOneSourceIntoStorage(
|
|
300144
|
+
return this.buildOneSourceIntoStorage({
|
|
300145
|
+
persistSource,
|
|
300146
|
+
instruction,
|
|
300147
|
+
manifest,
|
|
300148
|
+
environment,
|
|
300149
|
+
publicBuildSQL,
|
|
300150
|
+
buildSQL,
|
|
300151
|
+
builtEntries,
|
|
300152
|
+
dependsOnStorageUpstream,
|
|
300153
|
+
queryMetadata: runOptions.queryMetadata,
|
|
300154
|
+
incremental,
|
|
300155
|
+
contentSourceEntityId
|
|
300156
|
+
});
|
|
298446
300157
|
}
|
|
298447
300158
|
const dialect = persistSource.dialectName;
|
|
298448
300159
|
const quotedPhysicalPath = quoteTablePath(physicalTableName, dialect);
|
|
298449
300160
|
const lineage = incremental && contentSourceEntityId ? incrementalLineage({
|
|
298450
300161
|
declaration: incremental.declarations[persistSource.sourceID],
|
|
298451
300162
|
dialect,
|
|
300163
|
+
targetDialect: dialect,
|
|
298452
300164
|
physicalTableName,
|
|
298453
300165
|
connectionName: persistSource.connectionName,
|
|
298454
|
-
sourceEntityId: contentSourceEntityId
|
|
298455
|
-
isStorageBuild
|
|
300166
|
+
sourceEntityId: contentSourceEntityId
|
|
298456
300167
|
}) : undefined;
|
|
298457
300168
|
const incrementalRefresh = incremental && lineage && contentSourceEntityId ? {
|
|
298458
300169
|
context: incremental,
|
|
@@ -298463,10 +300174,13 @@ class MaterializationService {
|
|
|
298463
300174
|
...incrementalRefresh,
|
|
298464
300175
|
persistSource,
|
|
298465
300176
|
instruction,
|
|
298466
|
-
|
|
298467
|
-
|
|
298468
|
-
|
|
298469
|
-
|
|
300177
|
+
target: warehouseDeltaTarget({
|
|
300178
|
+
dialect,
|
|
300179
|
+
runner: (sql) => connection.runSQL(sql, runOptions),
|
|
300180
|
+
quotedTablePath: quotedPhysicalPath,
|
|
300181
|
+
lineage: incrementalRefresh.lineage,
|
|
300182
|
+
sourceSQL: buildSQL
|
|
300183
|
+
}),
|
|
298470
300184
|
manifest
|
|
298471
300185
|
});
|
|
298472
300186
|
if (applied)
|
|
@@ -298529,53 +300243,29 @@ class MaterializationService {
|
|
|
298529
300243
|
};
|
|
298530
300244
|
}
|
|
298531
300245
|
async refreshOneSourceIncrementally(params) {
|
|
298532
|
-
const { context, lineage, persistSource, instruction } = params;
|
|
300246
|
+
const { context, lineage, persistSource, instruction, target } = params;
|
|
298533
300247
|
const sourceEntityId = instruction.sourceEntityId;
|
|
298534
|
-
const dialect = persistSource.dialectName;
|
|
298535
|
-
const runner = (sql) => params.connection.runSQL(sql, params.runOptions);
|
|
298536
300248
|
const step = await planSourceRefresh({
|
|
298537
300249
|
context,
|
|
298538
300250
|
lineage,
|
|
298539
|
-
|
|
298540
|
-
quotedTablePath: params.quotedTablePath,
|
|
298541
|
-
sourceSQL: params.buildSQL,
|
|
300251
|
+
target,
|
|
298542
300252
|
columns: deriveColumns(persistSource).map((c) => String(c.name)),
|
|
298543
|
-
reseed: instruction.reseed
|
|
298544
|
-
runner
|
|
300253
|
+
reseed: instruction.reseed
|
|
298545
300254
|
});
|
|
298546
|
-
|
|
298547
|
-
|
|
298548
|
-
|
|
298549
|
-
|
|
298550
|
-
|
|
298551
|
-
|
|
298552
|
-
|
|
298553
|
-
|
|
298554
|
-
if (step.mode === "seed")
|
|
300255
|
+
const outcome = await applyIncrementalStep({
|
|
300256
|
+
context,
|
|
300257
|
+
lineage,
|
|
300258
|
+
step,
|
|
300259
|
+
target,
|
|
300260
|
+
sourceName: persistSource.name
|
|
300261
|
+
});
|
|
300262
|
+
if (!outcome.applied)
|
|
298555
300263
|
return;
|
|
298556
|
-
|
|
298557
|
-
|
|
298558
|
-
if (step.mode === "delta") {
|
|
298559
|
-
await applyDeltaScript(runner, dialect, step.statements);
|
|
298560
|
-
await advanceLedger({
|
|
298561
|
-
context,
|
|
298562
|
-
lineage,
|
|
298563
|
-
coveredThrough: step.coveredThrough
|
|
298564
|
-
});
|
|
298565
|
-
const durationMs = Math.round(performance.now() - startTime);
|
|
298566
|
-
appliedDurationMs = durationMs;
|
|
298567
|
-
recordSourceBuildDuration(durationMs, "delta");
|
|
298568
|
-
reportDeltaApplied({
|
|
298569
|
-
packageName: context.packageName,
|
|
298570
|
-
sourceName: persistSource.name,
|
|
298571
|
-
physicalTableName: lineage.physicalTableName,
|
|
298572
|
-
rangeStart: step.start.value,
|
|
298573
|
-
rangeEnd: step.end.value,
|
|
298574
|
-
durationMs
|
|
298575
|
-
});
|
|
300264
|
+
if (outcome.durationMs !== undefined) {
|
|
300265
|
+
recordSourceBuildDuration(outcome.durationMs, "delta");
|
|
298576
300266
|
}
|
|
298577
300267
|
params.manifest.update(sourceEntityId, {
|
|
298578
|
-
tableName:
|
|
300268
|
+
tableName: target.quotedTablePath
|
|
298579
300269
|
});
|
|
298580
300270
|
return {
|
|
298581
300271
|
sourceEntityId,
|
|
@@ -298585,18 +300275,61 @@ class MaterializationService {
|
|
|
298585
300275
|
connectionName: persistSource.connectionName,
|
|
298586
300276
|
realization: instruction.realization,
|
|
298587
300277
|
rowCount: null,
|
|
298588
|
-
buildDurationMs:
|
|
300278
|
+
buildDurationMs: outcome.durationMs ?? null,
|
|
298589
300279
|
queryCostBytes: null,
|
|
298590
|
-
...ledgerFields(lineage,
|
|
298591
|
-
...refreshFields(
|
|
300280
|
+
...ledgerFields(lineage, outcome.coveredThrough),
|
|
300281
|
+
...refreshFields(outcome.refresh)
|
|
298592
300282
|
};
|
|
298593
300283
|
}
|
|
298594
|
-
async buildOneSourceIntoStorage(
|
|
300284
|
+
async buildOneSourceIntoStorage(params) {
|
|
300285
|
+
const {
|
|
300286
|
+
persistSource,
|
|
300287
|
+
instruction,
|
|
300288
|
+
manifest,
|
|
300289
|
+
environment,
|
|
300290
|
+
builtEntries,
|
|
300291
|
+
dependsOnStorageUpstream,
|
|
300292
|
+
queryMetadata,
|
|
300293
|
+
incremental
|
|
300294
|
+
} = params;
|
|
298595
300295
|
const sourceEntityId = instruction.sourceEntityId;
|
|
298596
300296
|
const physicalTableName = instruction.physicalTableName;
|
|
298597
300297
|
const destinationName = instruction.destination;
|
|
298598
300298
|
const sourceConnection = environment.getApiConnection(persistSource.connectionName);
|
|
298599
300299
|
const destinationConnection = environment.getStorageDestination(destinationName);
|
|
300300
|
+
const lineage = incremental && params.contentSourceEntityId ? incrementalLineage({
|
|
300301
|
+
declaration: incremental.declarations[persistSource.sourceID],
|
|
300302
|
+
dialect: persistSource.dialectName,
|
|
300303
|
+
targetDialect: STORAGE_TARGET_DIALECT,
|
|
300304
|
+
physicalTableName,
|
|
300305
|
+
connectionName: persistSource.connectionName,
|
|
300306
|
+
storageDestinationName: destinationName,
|
|
300307
|
+
sourceEntityId: params.contentSourceEntityId
|
|
300308
|
+
}) : undefined;
|
|
300309
|
+
const chainedSeed = lineage !== undefined && dependsOnStorageUpstream;
|
|
300310
|
+
if (chainedSeed && lineage) {
|
|
300311
|
+
reportIncrementalStep({
|
|
300312
|
+
step: {
|
|
300313
|
+
mode: "seed",
|
|
300314
|
+
reasonCode: "chained_storage",
|
|
300315
|
+
reason: `the source reads a storage-materialized upstream, and a delta ` + `over a stored parent is not supported yet`
|
|
300316
|
+
},
|
|
300317
|
+
sourceName: persistSource.name,
|
|
300318
|
+
packageName: incremental.packageName,
|
|
300319
|
+
physicalTableName
|
|
300320
|
+
});
|
|
300321
|
+
await resetLedger(incremental, lineage);
|
|
300322
|
+
}
|
|
300323
|
+
const refresh = incremental && lineage && !chainedSeed ? this.storageRefreshFor({
|
|
300324
|
+
context: incremental,
|
|
300325
|
+
lineage,
|
|
300326
|
+
persistSource,
|
|
300327
|
+
instruction,
|
|
300328
|
+
destinationName,
|
|
300329
|
+
physicalTableName,
|
|
300330
|
+
buildSQL: params.buildSQL,
|
|
300331
|
+
queryMetadata
|
|
300332
|
+
}) : undefined;
|
|
298600
300333
|
const startTime = performance.now();
|
|
298601
300334
|
let result;
|
|
298602
300335
|
if (dependsOnStorageUpstream) {
|
|
@@ -298629,10 +300362,11 @@ class MaterializationService {
|
|
|
298629
300362
|
destinationName,
|
|
298630
300363
|
destinationConnection,
|
|
298631
300364
|
sourceConnection,
|
|
298632
|
-
buildSQL,
|
|
300365
|
+
buildSQL: params.publicBuildSQL,
|
|
298633
300366
|
physicalTableName,
|
|
298634
300367
|
environmentPath: environment.getEnvironmentPath(),
|
|
298635
|
-
queryMetadata
|
|
300368
|
+
queryMetadata,
|
|
300369
|
+
incremental: refresh
|
|
298636
300370
|
});
|
|
298637
300371
|
} catch (err) {
|
|
298638
300372
|
const safeDetail = redactConnectionSecrets(errMessage(err), sourceConnection, destinationConnection);
|
|
@@ -298657,6 +300391,8 @@ class MaterializationService {
|
|
|
298657
300391
|
schema: result.schema
|
|
298658
300392
|
});
|
|
298659
300393
|
} catch (gateErr) {
|
|
300394
|
+
if (result.refresh)
|
|
300395
|
+
throw gateErr;
|
|
298660
300396
|
try {
|
|
298661
300397
|
await dropStorageTable({
|
|
298662
300398
|
destinationName,
|
|
@@ -298676,12 +300412,17 @@ class MaterializationService {
|
|
|
298676
300412
|
}
|
|
298677
300413
|
manifest.update(sourceEntityId, { tableName: physicalTableName });
|
|
298678
300414
|
const durationMs = Math.round(performance.now() - startTime);
|
|
298679
|
-
|
|
298680
|
-
|
|
300415
|
+
if (result.refresh?.refresh === "delta") {
|
|
300416
|
+
recordSourceBuildDuration(durationMs, "delta_storage");
|
|
300417
|
+
} else if (!result.refresh) {
|
|
300418
|
+
recordSourceBuildDuration(durationMs, "storage");
|
|
300419
|
+
}
|
|
300420
|
+
logger.info(result.refresh ? `Refreshed materialized source ${persistSource.name} in storage` : `Built materialized source ${persistSource.name} into storage`, {
|
|
298681
300421
|
physicalTableName,
|
|
298682
300422
|
storageDestinationName: result.storageDestinationName,
|
|
298683
300423
|
columns: result.schema.length,
|
|
298684
300424
|
durationMs,
|
|
300425
|
+
refresh: result.refresh?.refresh,
|
|
298685
300426
|
readCost: result.readCost
|
|
298686
300427
|
});
|
|
298687
300428
|
return {
|
|
@@ -298694,10 +300435,62 @@ class MaterializationService {
|
|
|
298694
300435
|
schema: result.schema,
|
|
298695
300436
|
realization: instruction.realization,
|
|
298696
300437
|
rowCount: null,
|
|
298697
|
-
buildDurationMs: durationMs,
|
|
300438
|
+
buildDurationMs: result.refresh ? result.refresh.durationMs ?? null : durationMs,
|
|
300439
|
+
...ledgerFields(lineage, result.refresh?.coveredThrough ?? result.seededThrough),
|
|
300440
|
+
...refreshFields(result.refresh?.refresh ?? (lineage ? "full" : undefined)),
|
|
298698
300441
|
queryCostBytes: result.readCost?.bytesScanned ?? null
|
|
298699
300442
|
};
|
|
298700
300443
|
}
|
|
300444
|
+
storageRefreshFor(params) {
|
|
300445
|
+
const { context, lineage, persistSource, instruction } = params;
|
|
300446
|
+
return {
|
|
300447
|
+
plan: async ({ session, sourceType, handle, quotedTablePath }) => {
|
|
300448
|
+
const { target, readCost } = storageDeltaTarget({
|
|
300449
|
+
session,
|
|
300450
|
+
sourceType,
|
|
300451
|
+
handle,
|
|
300452
|
+
destinationName: params.destinationName,
|
|
300453
|
+
physicalTableName: params.physicalTableName,
|
|
300454
|
+
quotedTablePath,
|
|
300455
|
+
lineage,
|
|
300456
|
+
persistSource,
|
|
300457
|
+
buildSQL: params.buildSQL,
|
|
300458
|
+
queryMetadata: params.queryMetadata
|
|
300459
|
+
});
|
|
300460
|
+
const step = await planSourceRefresh({
|
|
300461
|
+
context,
|
|
300462
|
+
lineage,
|
|
300463
|
+
target,
|
|
300464
|
+
columns: deriveColumns(persistSource).map((c) => String(c.name)),
|
|
300465
|
+
reseed: instruction.reseed
|
|
300466
|
+
});
|
|
300467
|
+
const outcome = await applyIncrementalStep({
|
|
300468
|
+
context,
|
|
300469
|
+
lineage,
|
|
300470
|
+
step,
|
|
300471
|
+
target,
|
|
300472
|
+
sourceName: persistSource.name
|
|
300473
|
+
});
|
|
300474
|
+
if (!outcome.applied) {
|
|
300475
|
+
await resetLedger(context, lineage);
|
|
300476
|
+
return;
|
|
300477
|
+
}
|
|
300478
|
+
return {
|
|
300479
|
+
durationMs: outcome.durationMs,
|
|
300480
|
+
coveredThrough: outcome.coveredThrough,
|
|
300481
|
+
refresh: outcome.refresh ?? "none",
|
|
300482
|
+
readCost: readCost()
|
|
300483
|
+
};
|
|
300484
|
+
},
|
|
300485
|
+
afterSeed: ({ session, quotedTablePath }) => advanceLedgerAfterSeed({
|
|
300486
|
+
context,
|
|
300487
|
+
lineage,
|
|
300488
|
+
quotedTablePath,
|
|
300489
|
+
dialect: STORAGE_TARGET_DIALECT,
|
|
300490
|
+
runner: (sql) => session.runSQL(sql)
|
|
300491
|
+
})
|
|
300492
|
+
};
|
|
300493
|
+
}
|
|
298701
300494
|
async buildDownstreamViaParents(persistSource, destinationName, destinationConnection, builtEntries, environment, physicalTableName) {
|
|
298702
300495
|
const upstreams = deriveServeBindings(builtEntries).filter((b) => b.destinationName === destinationName);
|
|
298703
300496
|
if (upstreams.length === 0) {
|
|
@@ -299236,6 +301029,724 @@ function classifySpaFallback(requestPath, apiPrefix) {
|
|
|
299236
301029
|
};
|
|
299237
301030
|
}
|
|
299238
301031
|
|
|
301032
|
+
// ../../node_modules/express-rate-limit/dist/index.mjs
|
|
301033
|
+
var import_ip_address = __toESM(require_ip_address(), 1);
|
|
301034
|
+
var import_debug2 = __toESM(require_src5(), 1);
|
|
301035
|
+
import { isIPv6 } from "node:net";
|
|
301036
|
+
import { isIPv6 as isIPv62 } from "node:net";
|
|
301037
|
+
import { Buffer as Buffer3 } from "node:buffer";
|
|
301038
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
301039
|
+
import { isIP } from "node:net";
|
|
301040
|
+
var ipv4CompatibleSubnet = new import_ip_address.Address6("::/96");
|
|
301041
|
+
function ipKeyGenerator(ip, ipv6Subnet = 56) {
|
|
301042
|
+
if (isIPv6(ip)) {
|
|
301043
|
+
const address = new import_ip_address.Address6(ip);
|
|
301044
|
+
if (address.isMapped4() || address.is4() && address.isInSubnet(ipv4CompatibleSubnet))
|
|
301045
|
+
return address.to4().correctForm();
|
|
301046
|
+
if (ipv6Subnet) {
|
|
301047
|
+
const subnet = new import_ip_address.Address6(`${ip}/${ipv6Subnet}`);
|
|
301048
|
+
return subnet.networkForm();
|
|
301049
|
+
}
|
|
301050
|
+
}
|
|
301051
|
+
return ip;
|
|
301052
|
+
}
|
|
301053
|
+
var MemoryStore = class {
|
|
301054
|
+
constructor(validations2) {
|
|
301055
|
+
this.validations = validations2;
|
|
301056
|
+
this.previous = /* @__PURE__ */ new Map;
|
|
301057
|
+
this.current = /* @__PURE__ */ new Map;
|
|
301058
|
+
this.localKeys = true;
|
|
301059
|
+
}
|
|
301060
|
+
init(options) {
|
|
301061
|
+
this.windowMs = options.windowMs;
|
|
301062
|
+
this.validations?.windowMs(this.windowMs);
|
|
301063
|
+
if (this.interval)
|
|
301064
|
+
clearInterval(this.interval);
|
|
301065
|
+
this.interval = setInterval(() => {
|
|
301066
|
+
this.clearExpired();
|
|
301067
|
+
}, this.windowMs);
|
|
301068
|
+
this.interval.unref?.();
|
|
301069
|
+
}
|
|
301070
|
+
async get(key) {
|
|
301071
|
+
return this.current.get(key) ?? this.previous.get(key);
|
|
301072
|
+
}
|
|
301073
|
+
async increment(key) {
|
|
301074
|
+
const client = this.getClient(key);
|
|
301075
|
+
const now = Date.now();
|
|
301076
|
+
if (client.resetTime.getTime() <= now) {
|
|
301077
|
+
this.resetClient(client, now);
|
|
301078
|
+
}
|
|
301079
|
+
client.totalHits++;
|
|
301080
|
+
return client;
|
|
301081
|
+
}
|
|
301082
|
+
async decrement(key) {
|
|
301083
|
+
const client = this.getClient(key);
|
|
301084
|
+
if (client.totalHits > 0)
|
|
301085
|
+
client.totalHits--;
|
|
301086
|
+
}
|
|
301087
|
+
async resetKey(key) {
|
|
301088
|
+
this.current.delete(key);
|
|
301089
|
+
this.previous.delete(key);
|
|
301090
|
+
}
|
|
301091
|
+
async resetAll() {
|
|
301092
|
+
this.current.clear();
|
|
301093
|
+
this.previous.clear();
|
|
301094
|
+
}
|
|
301095
|
+
shutdown() {
|
|
301096
|
+
clearInterval(this.interval);
|
|
301097
|
+
this.resetAll();
|
|
301098
|
+
}
|
|
301099
|
+
resetClient(client, now = Date.now()) {
|
|
301100
|
+
client.totalHits = 0;
|
|
301101
|
+
client.resetTime.setTime(now + this.windowMs);
|
|
301102
|
+
return client;
|
|
301103
|
+
}
|
|
301104
|
+
getClient(key) {
|
|
301105
|
+
if (this.current.has(key))
|
|
301106
|
+
return this.current.get(key);
|
|
301107
|
+
let client;
|
|
301108
|
+
if (this.previous.has(key)) {
|
|
301109
|
+
client = this.previous.get(key);
|
|
301110
|
+
this.previous.delete(key);
|
|
301111
|
+
} else {
|
|
301112
|
+
client = { totalHits: 0, resetTime: /* @__PURE__ */ new Date };
|
|
301113
|
+
this.resetClient(client);
|
|
301114
|
+
}
|
|
301115
|
+
this.current.set(key, client);
|
|
301116
|
+
return client;
|
|
301117
|
+
}
|
|
301118
|
+
clearExpired() {
|
|
301119
|
+
this.previous = this.current;
|
|
301120
|
+
this.current = /* @__PURE__ */ new Map;
|
|
301121
|
+
}
|
|
301122
|
+
};
|
|
301123
|
+
var ConsoleLogger = {
|
|
301124
|
+
warn(...args) {
|
|
301125
|
+
console.warn(...args.reverse());
|
|
301126
|
+
},
|
|
301127
|
+
error(...args) {
|
|
301128
|
+
console.error(...args.reverse());
|
|
301129
|
+
}
|
|
301130
|
+
};
|
|
301131
|
+
var SUPPORTED_DRAFT_VERSIONS = [
|
|
301132
|
+
"draft-6",
|
|
301133
|
+
"draft-7",
|
|
301134
|
+
"draft-8"
|
|
301135
|
+
];
|
|
301136
|
+
var getResetSeconds = (windowMs, resetTime) => {
|
|
301137
|
+
let resetSeconds;
|
|
301138
|
+
if (resetTime) {
|
|
301139
|
+
const deltaSeconds = Math.ceil((resetTime.getTime() - Date.now()) / 1000);
|
|
301140
|
+
resetSeconds = Math.max(0, deltaSeconds);
|
|
301141
|
+
} else {
|
|
301142
|
+
resetSeconds = Math.ceil(windowMs / 1000);
|
|
301143
|
+
}
|
|
301144
|
+
return resetSeconds;
|
|
301145
|
+
};
|
|
301146
|
+
var getPartitionKey = (key) => {
|
|
301147
|
+
const hash = createHash4("sha256");
|
|
301148
|
+
hash.update(key);
|
|
301149
|
+
const partitionKey = hash.digest("hex").slice(0, 12);
|
|
301150
|
+
return Buffer3.from(partitionKey).toString("base64");
|
|
301151
|
+
};
|
|
301152
|
+
var setLegacyHeaders = (response, info) => {
|
|
301153
|
+
if (response.headersSent)
|
|
301154
|
+
return;
|
|
301155
|
+
response.setHeader("X-RateLimit-Limit", info.limit.toString());
|
|
301156
|
+
response.setHeader("X-RateLimit-Remaining", info.remaining.toString());
|
|
301157
|
+
if (info.resetTime instanceof Date) {
|
|
301158
|
+
response.setHeader("Date", (/* @__PURE__ */ new Date()).toUTCString());
|
|
301159
|
+
response.setHeader("X-RateLimit-Reset", Math.ceil(info.resetTime.getTime() / 1000).toString());
|
|
301160
|
+
}
|
|
301161
|
+
};
|
|
301162
|
+
var setDraft6Headers = (response, info, windowMs) => {
|
|
301163
|
+
if (response.headersSent)
|
|
301164
|
+
return;
|
|
301165
|
+
const windowSeconds = Math.ceil(windowMs / 1000);
|
|
301166
|
+
const resetSeconds = getResetSeconds(windowMs, info.resetTime);
|
|
301167
|
+
response.setHeader("RateLimit-Policy", `${info.limit};w=${windowSeconds}`);
|
|
301168
|
+
response.setHeader("RateLimit-Limit", info.limit.toString());
|
|
301169
|
+
response.setHeader("RateLimit-Remaining", info.remaining.toString());
|
|
301170
|
+
if (typeof resetSeconds === "number")
|
|
301171
|
+
response.setHeader("RateLimit-Reset", resetSeconds.toString());
|
|
301172
|
+
};
|
|
301173
|
+
var setDraft7Headers = (response, info, windowMs) => {
|
|
301174
|
+
if (response.headersSent)
|
|
301175
|
+
return;
|
|
301176
|
+
const windowSeconds = Math.ceil(windowMs / 1000);
|
|
301177
|
+
const resetSeconds = getResetSeconds(windowMs, info.resetTime);
|
|
301178
|
+
response.setHeader("RateLimit-Policy", `${info.limit};w=${windowSeconds}`);
|
|
301179
|
+
response.setHeader("RateLimit", `limit=${info.limit}, remaining=${info.remaining}, reset=${resetSeconds}`);
|
|
301180
|
+
};
|
|
301181
|
+
var setDraft8Headers = (response, info, windowMs, name, key) => {
|
|
301182
|
+
if (response.headersSent)
|
|
301183
|
+
return;
|
|
301184
|
+
const windowSeconds = Math.ceil(windowMs / 1000);
|
|
301185
|
+
const resetSeconds = getResetSeconds(windowMs, info.resetTime);
|
|
301186
|
+
const partitionKey = getPartitionKey(key);
|
|
301187
|
+
const header = `r=${info.remaining}; t=${resetSeconds}`;
|
|
301188
|
+
const policy = `q=${info.limit}; w=${windowSeconds}; pk=:${partitionKey}:`;
|
|
301189
|
+
response.append("RateLimit", `"${name}"; ${header}`);
|
|
301190
|
+
response.append("RateLimit-Policy", `"${name}"; ${policy}`);
|
|
301191
|
+
};
|
|
301192
|
+
var setRetryAfterHeader = (response, info, windowMs) => {
|
|
301193
|
+
if (response.headersSent)
|
|
301194
|
+
return;
|
|
301195
|
+
const resetSeconds = getResetSeconds(windowMs, info.resetTime);
|
|
301196
|
+
response.setHeader("Retry-After", resetSeconds.toString());
|
|
301197
|
+
};
|
|
301198
|
+
var omitUndefinedProperties = (passedOptions) => {
|
|
301199
|
+
const omittedOptions = {};
|
|
301200
|
+
for (const k of Object.keys(passedOptions)) {
|
|
301201
|
+
const key = k;
|
|
301202
|
+
if (passedOptions[key] !== undefined) {
|
|
301203
|
+
omittedOptions[key] = passedOptions[key];
|
|
301204
|
+
}
|
|
301205
|
+
}
|
|
301206
|
+
return omittedOptions;
|
|
301207
|
+
};
|
|
301208
|
+
var ValidationError = class extends Error {
|
|
301209
|
+
constructor(code, message) {
|
|
301210
|
+
const url2 = `https://express-rate-limit.github.io/${code}/`;
|
|
301211
|
+
super(`${message} See ${url2} for more information.`);
|
|
301212
|
+
this.name = this.constructor.name;
|
|
301213
|
+
this.code = code;
|
|
301214
|
+
this.help = url2;
|
|
301215
|
+
}
|
|
301216
|
+
};
|
|
301217
|
+
var ChangeWarning = class extends ValidationError {
|
|
301218
|
+
};
|
|
301219
|
+
var usedStores = /* @__PURE__ */ new Set;
|
|
301220
|
+
var singleCountKeys = /* @__PURE__ */ new WeakMap;
|
|
301221
|
+
var validations = {
|
|
301222
|
+
enabled: {
|
|
301223
|
+
default: true
|
|
301224
|
+
},
|
|
301225
|
+
disable() {
|
|
301226
|
+
for (const k of Object.keys(this.enabled))
|
|
301227
|
+
this.enabled[k] = false;
|
|
301228
|
+
},
|
|
301229
|
+
ip(ip) {
|
|
301230
|
+
if (ip === undefined) {
|
|
301231
|
+
throw new ValidationError("ERR_ERL_UNDEFINED_IP_ADDRESS", `An undefined 'request.ip' was detected. This might indicate a misconfiguration or the connection being destroyed prematurely.`);
|
|
301232
|
+
}
|
|
301233
|
+
if (!isIP(ip)) {
|
|
301234
|
+
throw new ValidationError("ERR_ERL_INVALID_IP_ADDRESS", `An invalid 'request.ip' (${ip}) was detected. Consider passing a custom 'keyGenerator' function to the rate limiter.`);
|
|
301235
|
+
}
|
|
301236
|
+
},
|
|
301237
|
+
trustProxy(request) {
|
|
301238
|
+
if (request.app.get("trust proxy") === true) {
|
|
301239
|
+
throw new ValidationError("ERR_ERL_PERMISSIVE_TRUST_PROXY", `The Express 'trust proxy' setting is true, which allows anyone to trivially bypass IP-based rate limiting.`);
|
|
301240
|
+
}
|
|
301241
|
+
},
|
|
301242
|
+
xForwardedForHeader(request) {
|
|
301243
|
+
if (request.headers["x-forwarded-for"] && request.app.get("trust proxy") === false) {
|
|
301244
|
+
throw new ValidationError("ERR_ERL_UNEXPECTED_X_FORWARDED_FOR", `The 'X-Forwarded-For' header is set but the Express 'trust proxy' setting is false (default). This could indicate a misconfiguration which would prevent express-rate-limit from accurately identifying users.`);
|
|
301245
|
+
}
|
|
301246
|
+
},
|
|
301247
|
+
forwardedHeader(request) {
|
|
301248
|
+
if (request.headers.forwarded && request.ip === request.socket?.remoteAddress) {
|
|
301249
|
+
throw new ValidationError("ERR_ERL_FORWARDED_HEADER", `The 'Forwarded' header (standardized X-Forwarded-For) is set but currently being ignored. Add a custom keyGenerator to use a value from this header.`);
|
|
301250
|
+
}
|
|
301251
|
+
},
|
|
301252
|
+
positiveHits(hits) {
|
|
301253
|
+
if (typeof hits !== "number" || hits < 1 || hits !== Math.round(hits)) {
|
|
301254
|
+
throw new ValidationError("ERR_ERL_INVALID_HITS", `The totalHits value returned from the store must be a positive integer, got ${hits}`);
|
|
301255
|
+
}
|
|
301256
|
+
},
|
|
301257
|
+
unsharedStore(store) {
|
|
301258
|
+
if (usedStores.has(store)) {
|
|
301259
|
+
const maybeUniquePrefix = store?.localKeys ? "" : " (with a unique prefix)";
|
|
301260
|
+
throw new ValidationError("ERR_ERL_STORE_REUSE", `A Store instance must not be shared across multiple rate limiters. Create a new instance of ${store.constructor.name}${maybeUniquePrefix} for each limiter instead.`);
|
|
301261
|
+
}
|
|
301262
|
+
usedStores.add(store);
|
|
301263
|
+
},
|
|
301264
|
+
singleCount(request, store, key) {
|
|
301265
|
+
let storeKeys = singleCountKeys.get(request);
|
|
301266
|
+
if (!storeKeys) {
|
|
301267
|
+
storeKeys = /* @__PURE__ */ new Map;
|
|
301268
|
+
singleCountKeys.set(request, storeKeys);
|
|
301269
|
+
}
|
|
301270
|
+
const storeKey = store.localKeys ? store : store.constructor.name;
|
|
301271
|
+
let keys = storeKeys.get(storeKey);
|
|
301272
|
+
if (!keys) {
|
|
301273
|
+
keys = [];
|
|
301274
|
+
storeKeys.set(storeKey, keys);
|
|
301275
|
+
}
|
|
301276
|
+
const prefixedKey = `${store.prefix ?? ""}${key}`;
|
|
301277
|
+
if (keys.includes(prefixedKey)) {
|
|
301278
|
+
throw new ValidationError("ERR_ERL_DOUBLE_COUNT", `The hit count for ${key} was incremented more than once for a single request.`);
|
|
301279
|
+
}
|
|
301280
|
+
keys.push(prefixedKey);
|
|
301281
|
+
},
|
|
301282
|
+
limit(limit) {
|
|
301283
|
+
if (limit === 0) {
|
|
301284
|
+
throw new ChangeWarning("WRN_ERL_MAX_ZERO", "Setting limit or max to 0 disables rate limiting in express-rate-limit v6 and older, but will cause all requests to be blocked in v7");
|
|
301285
|
+
}
|
|
301286
|
+
},
|
|
301287
|
+
draftPolliHeaders(draft_polli_ratelimit_headers) {
|
|
301288
|
+
if (draft_polli_ratelimit_headers) {
|
|
301289
|
+
throw new ChangeWarning("WRN_ERL_DEPRECATED_DRAFT_POLLI_HEADERS", `The draft_polli_ratelimit_headers configuration option is deprecated and has been removed in express-rate-limit v7, please set standardHeaders: 'draft-6' instead.`);
|
|
301290
|
+
}
|
|
301291
|
+
},
|
|
301292
|
+
onLimitReached(onLimitReached) {
|
|
301293
|
+
if (onLimitReached) {
|
|
301294
|
+
throw new ChangeWarning("WRN_ERL_DEPRECATED_ON_LIMIT_REACHED", "The onLimitReached configuration option is deprecated and has been removed in express-rate-limit v7.");
|
|
301295
|
+
}
|
|
301296
|
+
},
|
|
301297
|
+
headersDraftVersion(version) {
|
|
301298
|
+
if (typeof version !== "string" || !SUPPORTED_DRAFT_VERSIONS.includes(version)) {
|
|
301299
|
+
const versionString = SUPPORTED_DRAFT_VERSIONS.join(", ");
|
|
301300
|
+
throw new ValidationError("ERR_ERL_HEADERS_UNSUPPORTED_DRAFT_VERSION", `standardHeaders: only the following versions of the IETF draft specification are supported: ${versionString}.`);
|
|
301301
|
+
}
|
|
301302
|
+
},
|
|
301303
|
+
headersResetTime(resetTime) {
|
|
301304
|
+
if (!resetTime) {
|
|
301305
|
+
throw new ValidationError("ERR_ERL_HEADERS_NO_RESET", `standardHeaders: 'draft-7' requires a 'resetTime', but the store did not provide one. The 'windowMs' value will be used instead, which may cause clients to wait longer than necessary.`);
|
|
301306
|
+
}
|
|
301307
|
+
},
|
|
301308
|
+
knownOptions(passedOptions) {
|
|
301309
|
+
if (!passedOptions)
|
|
301310
|
+
return;
|
|
301311
|
+
const optionsMap = {
|
|
301312
|
+
windowMs: true,
|
|
301313
|
+
limit: true,
|
|
301314
|
+
message: true,
|
|
301315
|
+
statusCode: true,
|
|
301316
|
+
legacyHeaders: true,
|
|
301317
|
+
standardHeaders: true,
|
|
301318
|
+
identifier: true,
|
|
301319
|
+
requestPropertyName: true,
|
|
301320
|
+
skipFailedRequests: true,
|
|
301321
|
+
skipSuccessfulRequests: true,
|
|
301322
|
+
keyGenerator: true,
|
|
301323
|
+
ipv6Subnet: true,
|
|
301324
|
+
handler: true,
|
|
301325
|
+
skip: true,
|
|
301326
|
+
requestWasSuccessful: true,
|
|
301327
|
+
store: true,
|
|
301328
|
+
validate: true,
|
|
301329
|
+
headers: true,
|
|
301330
|
+
max: true,
|
|
301331
|
+
passOnStoreError: true,
|
|
301332
|
+
logger: true
|
|
301333
|
+
};
|
|
301334
|
+
const validOptions = Object.keys(optionsMap).concat("draft_polli_ratelimit_headers", "delayAfter", "delayMs", "maxDelayMs");
|
|
301335
|
+
for (const key of Object.keys(passedOptions)) {
|
|
301336
|
+
if (!validOptions.includes(key)) {
|
|
301337
|
+
throw new ValidationError("ERR_ERL_UNKNOWN_OPTION", `Unexpected configuration option: ${key}`);
|
|
301338
|
+
}
|
|
301339
|
+
}
|
|
301340
|
+
},
|
|
301341
|
+
validationsConfig() {
|
|
301342
|
+
const supportedValidations = Object.keys(this).filter((k) => !["enabled", "disable"].includes(k));
|
|
301343
|
+
supportedValidations.push("default");
|
|
301344
|
+
for (const key of Object.keys(this.enabled)) {
|
|
301345
|
+
if (!supportedValidations.includes(key)) {
|
|
301346
|
+
throw new ValidationError("ERR_ERL_UNKNOWN_VALIDATION", `options.validate.${key} is not recognized. Supported validate options are: ${supportedValidations.join(", ")}.`);
|
|
301347
|
+
}
|
|
301348
|
+
}
|
|
301349
|
+
},
|
|
301350
|
+
creationStack(store) {
|
|
301351
|
+
const { stack } = new Error("express-rate-limit validation check (set options.validate.creationStack=false to disable)");
|
|
301352
|
+
if (stack?.includes("Layer.handle [as handle_request]") || stack?.includes("Layer.handleRequest")) {
|
|
301353
|
+
if (!store.localKeys) {
|
|
301354
|
+
throw new ValidationError("ERR_ERL_CREATED_IN_REQUEST_HANDLER", "express-rate-limit instance should *usually* be created at app initialization, not when responding to a request.");
|
|
301355
|
+
}
|
|
301356
|
+
throw new ValidationError("ERR_ERL_CREATED_IN_REQUEST_HANDLER", "express-rate-limit instance should be created at app initialization, not when responding to a request.");
|
|
301357
|
+
}
|
|
301358
|
+
},
|
|
301359
|
+
ipv6Subnet(ipv6Subnet) {
|
|
301360
|
+
if (ipv6Subnet === false) {
|
|
301361
|
+
return;
|
|
301362
|
+
}
|
|
301363
|
+
if (!Number.isInteger(ipv6Subnet) || ipv6Subnet < 32 || ipv6Subnet > 64) {
|
|
301364
|
+
throw new ValidationError("ERR_ERL_IPV6_SUBNET", `Unexpected ipv6Subnet value: ${ipv6Subnet}. Expected an integer between 32 and 64 (usually 48-64).`);
|
|
301365
|
+
}
|
|
301366
|
+
},
|
|
301367
|
+
ipv6SubnetOrKeyGenerator(options) {
|
|
301368
|
+
if (options.ipv6Subnet !== undefined && options.keyGenerator) {
|
|
301369
|
+
throw new ValidationError("ERR_ERL_IPV6SUBNET_OR_KEYGENERATOR", `Incompatible options: the 'ipv6Subnet' option is ignored when a custom 'keyGenerator' function is also set.`);
|
|
301370
|
+
}
|
|
301371
|
+
},
|
|
301372
|
+
keyGeneratorIpFallback(keyGenerator) {
|
|
301373
|
+
if (!keyGenerator) {
|
|
301374
|
+
return;
|
|
301375
|
+
}
|
|
301376
|
+
const src = keyGenerator.toString();
|
|
301377
|
+
if ((src.includes("req.ip") || src.includes("request.ip")) && !src.includes("ipKeyGenerator")) {
|
|
301378
|
+
throw new ValidationError("ERR_ERL_KEY_GEN_IPV6", "Custom keyGenerator appears to use request IP without calling the ipKeyGenerator helper function for IPv6 addresses. This could allow IPv6 users to bypass limits.");
|
|
301379
|
+
}
|
|
301380
|
+
},
|
|
301381
|
+
windowMs(windowMs) {
|
|
301382
|
+
const SET_TIMEOUT_MAX = 2 ** 31 - 1;
|
|
301383
|
+
if (typeof windowMs !== "number" || Number.isNaN(windowMs) || windowMs < 1 || windowMs > SET_TIMEOUT_MAX) {
|
|
301384
|
+
throw new ValidationError("ERR_ERL_WINDOW_MS", `Invalid windowMs value: ${windowMs}${typeof windowMs !== "number" ? ` (${typeof windowMs})` : ""}, must be a number between 1 and ${SET_TIMEOUT_MAX} when using the default MemoryStore`);
|
|
301385
|
+
}
|
|
301386
|
+
}
|
|
301387
|
+
};
|
|
301388
|
+
function validateLogger(logger2) {
|
|
301389
|
+
if (typeof logger2 !== "object" || typeof logger2.error !== "function" || typeof logger2.warn !== "function") {
|
|
301390
|
+
throw new TypeError("Provided logger does not implement the Logger interface");
|
|
301391
|
+
}
|
|
301392
|
+
}
|
|
301393
|
+
var getValidations = (_enabled, logger2) => {
|
|
301394
|
+
validateLogger(logger2);
|
|
301395
|
+
let enabled;
|
|
301396
|
+
if (typeof _enabled === "boolean") {
|
|
301397
|
+
enabled = {
|
|
301398
|
+
default: _enabled
|
|
301399
|
+
};
|
|
301400
|
+
} else {
|
|
301401
|
+
enabled = {
|
|
301402
|
+
default: true,
|
|
301403
|
+
..._enabled
|
|
301404
|
+
};
|
|
301405
|
+
}
|
|
301406
|
+
const wrappedValidations = { enabled };
|
|
301407
|
+
for (const [name, validation] of Object.entries(validations)) {
|
|
301408
|
+
if (typeof validation === "function")
|
|
301409
|
+
wrappedValidations[name] = (...args) => {
|
|
301410
|
+
if (!(enabled[name] ?? enabled.default)) {
|
|
301411
|
+
return;
|
|
301412
|
+
}
|
|
301413
|
+
enabled[name] = false;
|
|
301414
|
+
try {
|
|
301415
|
+
validation.apply(wrappedValidations, args);
|
|
301416
|
+
} catch (error) {
|
|
301417
|
+
if (error instanceof ChangeWarning)
|
|
301418
|
+
logger2.warn(error);
|
|
301419
|
+
else
|
|
301420
|
+
logger2.error(error);
|
|
301421
|
+
}
|
|
301422
|
+
};
|
|
301423
|
+
}
|
|
301424
|
+
const inspect = /* @__PURE__ */ Symbol.for("nodejs.util.inspect.custom");
|
|
301425
|
+
if (inspect)
|
|
301426
|
+
wrappedValidations[inspect] = () => wrappedValidations.enabled;
|
|
301427
|
+
return wrappedValidations;
|
|
301428
|
+
};
|
|
301429
|
+
var isLegacyStore = (store) => typeof store.incr === "function" && typeof store.increment !== "function";
|
|
301430
|
+
var promisifyStore = (passedStore) => {
|
|
301431
|
+
if (!isLegacyStore(passedStore)) {
|
|
301432
|
+
return passedStore;
|
|
301433
|
+
}
|
|
301434
|
+
const legacyStore = passedStore;
|
|
301435
|
+
|
|
301436
|
+
class PromisifiedStore {
|
|
301437
|
+
async increment(key) {
|
|
301438
|
+
return new Promise((resolve6, reject) => {
|
|
301439
|
+
legacyStore.incr(key, (error, totalHits, resetTime) => {
|
|
301440
|
+
if (error)
|
|
301441
|
+
reject(error);
|
|
301442
|
+
resolve6({ totalHits, resetTime });
|
|
301443
|
+
});
|
|
301444
|
+
});
|
|
301445
|
+
}
|
|
301446
|
+
async decrement(key) {
|
|
301447
|
+
return legacyStore.decrement(key);
|
|
301448
|
+
}
|
|
301449
|
+
async resetKey(key) {
|
|
301450
|
+
return legacyStore.resetKey(key);
|
|
301451
|
+
}
|
|
301452
|
+
async resetAll() {
|
|
301453
|
+
if (typeof legacyStore.resetAll === "function")
|
|
301454
|
+
return legacyStore.resetAll();
|
|
301455
|
+
}
|
|
301456
|
+
}
|
|
301457
|
+
return new PromisifiedStore;
|
|
301458
|
+
};
|
|
301459
|
+
var getOptionsFromConfig = (config) => {
|
|
301460
|
+
const { validations: validations2, ...directlyPassableEntries } = config;
|
|
301461
|
+
return {
|
|
301462
|
+
...directlyPassableEntries,
|
|
301463
|
+
validate: validations2.enabled
|
|
301464
|
+
};
|
|
301465
|
+
};
|
|
301466
|
+
var parseOptions = (passedOptions) => {
|
|
301467
|
+
const notUndefinedOptions = omitUndefinedProperties(passedOptions);
|
|
301468
|
+
const logger2 = passedOptions.logger ?? ConsoleLogger;
|
|
301469
|
+
const validations2 = getValidations(notUndefinedOptions?.validate ?? true, logger2);
|
|
301470
|
+
validations2.validationsConfig();
|
|
301471
|
+
validations2.knownOptions(passedOptions);
|
|
301472
|
+
validations2.draftPolliHeaders(notUndefinedOptions.draft_polli_ratelimit_headers);
|
|
301473
|
+
validations2.onLimitReached(notUndefinedOptions.onLimitReached);
|
|
301474
|
+
if (notUndefinedOptions.ipv6Subnet !== undefined && typeof notUndefinedOptions.ipv6Subnet !== "function") {
|
|
301475
|
+
validations2.ipv6Subnet(notUndefinedOptions.ipv6Subnet);
|
|
301476
|
+
}
|
|
301477
|
+
validations2.keyGeneratorIpFallback(notUndefinedOptions.keyGenerator);
|
|
301478
|
+
validations2.ipv6SubnetOrKeyGenerator(notUndefinedOptions);
|
|
301479
|
+
let standardHeaders = notUndefinedOptions.standardHeaders ?? false;
|
|
301480
|
+
if (standardHeaders === true)
|
|
301481
|
+
standardHeaders = "draft-6";
|
|
301482
|
+
const config = {
|
|
301483
|
+
windowMs: 60 * 1000,
|
|
301484
|
+
limit: passedOptions.max ?? 5,
|
|
301485
|
+
message: "Too many requests, please try again later.",
|
|
301486
|
+
statusCode: 429,
|
|
301487
|
+
legacyHeaders: passedOptions.headers ?? true,
|
|
301488
|
+
identifier(request, _response) {
|
|
301489
|
+
let duration = "";
|
|
301490
|
+
const property = config.requestPropertyName;
|
|
301491
|
+
const { limit } = request[property];
|
|
301492
|
+
const seconds = config.windowMs / 1000;
|
|
301493
|
+
const minutes = config.windowMs / (1000 * 60);
|
|
301494
|
+
const hours = config.windowMs / (1000 * 60 * 60);
|
|
301495
|
+
const days = config.windowMs / (1000 * 60 * 60 * 24);
|
|
301496
|
+
if (seconds < 60)
|
|
301497
|
+
duration = `${seconds}sec`;
|
|
301498
|
+
else if (minutes < 60)
|
|
301499
|
+
duration = `${minutes}min`;
|
|
301500
|
+
else if (hours < 24)
|
|
301501
|
+
duration = `${hours}hr${hours > 1 ? "s" : ""}`;
|
|
301502
|
+
else
|
|
301503
|
+
duration = `${days}day${days > 1 ? "s" : ""}`;
|
|
301504
|
+
return `${limit}-in-${duration}`;
|
|
301505
|
+
},
|
|
301506
|
+
requestPropertyName: "rateLimit",
|
|
301507
|
+
skipFailedRequests: false,
|
|
301508
|
+
skipSuccessfulRequests: false,
|
|
301509
|
+
requestWasSuccessful: (_request, response) => response.statusCode < 400,
|
|
301510
|
+
skip: (_request, _response) => false,
|
|
301511
|
+
async keyGenerator(request, response) {
|
|
301512
|
+
validations2.ip(request.ip);
|
|
301513
|
+
validations2.trustProxy(request);
|
|
301514
|
+
validations2.xForwardedForHeader(request);
|
|
301515
|
+
validations2.forwardedHeader(request);
|
|
301516
|
+
const ip = request.ip;
|
|
301517
|
+
let subnet = 56;
|
|
301518
|
+
if (isIPv62(ip)) {
|
|
301519
|
+
subnet = typeof config.ipv6Subnet === "function" ? await config.ipv6Subnet(request, response) : config.ipv6Subnet;
|
|
301520
|
+
if (typeof config.ipv6Subnet === "function")
|
|
301521
|
+
validations2.ipv6Subnet(subnet);
|
|
301522
|
+
}
|
|
301523
|
+
return ipKeyGenerator(ip, subnet);
|
|
301524
|
+
},
|
|
301525
|
+
ipv6Subnet: 56,
|
|
301526
|
+
async handler(request, response, _next, _optionsUsed) {
|
|
301527
|
+
response.status(config.statusCode);
|
|
301528
|
+
const message = typeof config.message === "function" ? await config.message(request, response) : config.message;
|
|
301529
|
+
if (!response.writableEnded)
|
|
301530
|
+
response.send(message);
|
|
301531
|
+
},
|
|
301532
|
+
passOnStoreError: false,
|
|
301533
|
+
...notUndefinedOptions,
|
|
301534
|
+
standardHeaders,
|
|
301535
|
+
store: promisifyStore(notUndefinedOptions.store ?? new MemoryStore(validations2)),
|
|
301536
|
+
validations: validations2,
|
|
301537
|
+
logger: logger2
|
|
301538
|
+
};
|
|
301539
|
+
if (typeof config.store.increment !== "function" || typeof config.store.decrement !== "function" || typeof config.store.resetKey !== "function" || config.store.resetAll !== undefined && typeof config.store.resetAll !== "function" || config.store.init !== undefined && typeof config.store.init !== "function") {
|
|
301540
|
+
throw new TypeError("An invalid store was passed. Please ensure that the store is a class that implements the `Store` interface.");
|
|
301541
|
+
}
|
|
301542
|
+
return config;
|
|
301543
|
+
};
|
|
301544
|
+
var handleAsyncErrors = (fn) => async (request, response, next) => {
|
|
301545
|
+
try {
|
|
301546
|
+
await Promise.resolve(fn(request, response, next)).catch(next);
|
|
301547
|
+
} catch (error) {
|
|
301548
|
+
next(error);
|
|
301549
|
+
}
|
|
301550
|
+
};
|
|
301551
|
+
var rateLimit = (passedOptions) => {
|
|
301552
|
+
const config = parseOptions(passedOptions ?? {});
|
|
301553
|
+
const options = getOptionsFromConfig(config);
|
|
301554
|
+
const debug2 = import_debug2.default("express-rate-limit");
|
|
301555
|
+
debug2("creating new rate limiter with %o", config.store.constructor.name);
|
|
301556
|
+
for (const [key, val] of Object.entries(config))
|
|
301557
|
+
debug2("set %s to %o", key, val);
|
|
301558
|
+
config.validations.creationStack(config.store);
|
|
301559
|
+
config.validations.unsharedStore(config.store);
|
|
301560
|
+
if (typeof config.store.init === "function") {
|
|
301561
|
+
debug2("executing init for store");
|
|
301562
|
+
try {
|
|
301563
|
+
const storeInit = config.store.init(options);
|
|
301564
|
+
if (storeInit instanceof Promise) {
|
|
301565
|
+
storeInit.catch((error) => config.logger.error(error, "express-rate-limit: async error during store initialization."));
|
|
301566
|
+
}
|
|
301567
|
+
} catch (error) {
|
|
301568
|
+
config.logger.error(error, "express-rate-limit: error during store initialization.");
|
|
301569
|
+
}
|
|
301570
|
+
}
|
|
301571
|
+
const middleware = handleAsyncErrors(async (request, response, next) => {
|
|
301572
|
+
const closePromise = config.skipFailedRequests && new Promise((resolve6) => response.once("close", resolve6));
|
|
301573
|
+
const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((resolve6) => response.once("finish", resolve6));
|
|
301574
|
+
const errorPromise = config.skipFailedRequests && new Promise((resolve6) => response.once("error", resolve6));
|
|
301575
|
+
debug2("requested %o", request.originalUrl);
|
|
301576
|
+
debug2("request from ip %o", request.ip);
|
|
301577
|
+
const skip = await config.skip(request, response);
|
|
301578
|
+
if (skip) {
|
|
301579
|
+
debug2("skipping request");
|
|
301580
|
+
next();
|
|
301581
|
+
return;
|
|
301582
|
+
}
|
|
301583
|
+
const augmentedRequest = request;
|
|
301584
|
+
const key = await config.keyGenerator(request, response);
|
|
301585
|
+
debug2("computed key %o", key);
|
|
301586
|
+
debug2("incrementing count");
|
|
301587
|
+
let totalHits = 0;
|
|
301588
|
+
let resetTime;
|
|
301589
|
+
try {
|
|
301590
|
+
const incrementResult = await config.store.increment(key);
|
|
301591
|
+
totalHits = incrementResult.totalHits;
|
|
301592
|
+
resetTime = incrementResult.resetTime;
|
|
301593
|
+
} catch (error) {
|
|
301594
|
+
if (config.passOnStoreError) {
|
|
301595
|
+
config.logger.error(error, "express-rate-limit: error from store, allowing request without rate-limiting.");
|
|
301596
|
+
next();
|
|
301597
|
+
return;
|
|
301598
|
+
}
|
|
301599
|
+
throw error;
|
|
301600
|
+
}
|
|
301601
|
+
config.validations.positiveHits(totalHits);
|
|
301602
|
+
config.validations.singleCount(request, config.store, key);
|
|
301603
|
+
const retrieveLimit = typeof config.limit === "function" ? config.limit(request, response) : config.limit;
|
|
301604
|
+
const limit = await retrieveLimit;
|
|
301605
|
+
config.validations.limit(limit);
|
|
301606
|
+
const info = {
|
|
301607
|
+
limit,
|
|
301608
|
+
used: totalHits,
|
|
301609
|
+
remaining: Math.max(limit - totalHits, 0),
|
|
301610
|
+
resetTime,
|
|
301611
|
+
key
|
|
301612
|
+
};
|
|
301613
|
+
for (const [key2, val] of Object.entries(info))
|
|
301614
|
+
debug2("set request.%s.%s to be %o", config.requestPropertyName, key2, val);
|
|
301615
|
+
Object.defineProperty(info, "current", {
|
|
301616
|
+
configurable: false,
|
|
301617
|
+
enumerable: false,
|
|
301618
|
+
value: totalHits
|
|
301619
|
+
});
|
|
301620
|
+
augmentedRequest[config.requestPropertyName] = info;
|
|
301621
|
+
if (config.legacyHeaders && !response.headersSent) {
|
|
301622
|
+
debug2("set legacy headers");
|
|
301623
|
+
setLegacyHeaders(response, info);
|
|
301624
|
+
}
|
|
301625
|
+
if (config.standardHeaders && !response.headersSent) {
|
|
301626
|
+
switch (config.standardHeaders) {
|
|
301627
|
+
case "draft-6": {
|
|
301628
|
+
debug2("set ietf draft 6 headers");
|
|
301629
|
+
setDraft6Headers(response, info, config.windowMs);
|
|
301630
|
+
break;
|
|
301631
|
+
}
|
|
301632
|
+
case "draft-7": {
|
|
301633
|
+
debug2("set ietf draft 7 headers");
|
|
301634
|
+
config.validations.headersResetTime(info.resetTime);
|
|
301635
|
+
setDraft7Headers(response, info, config.windowMs);
|
|
301636
|
+
break;
|
|
301637
|
+
}
|
|
301638
|
+
case "draft-8": {
|
|
301639
|
+
const retrieveName = typeof config.identifier === "function" ? config.identifier(request, response) : config.identifier;
|
|
301640
|
+
const name = await retrieveName;
|
|
301641
|
+
debug2("set ietf draft 8 headers");
|
|
301642
|
+
debug2("set name to %o", name);
|
|
301643
|
+
config.validations.headersResetTime(info.resetTime);
|
|
301644
|
+
setDraft8Headers(response, info, config.windowMs, name, key);
|
|
301645
|
+
break;
|
|
301646
|
+
}
|
|
301647
|
+
default: {
|
|
301648
|
+
config.validations.headersDraftVersion(config.standardHeaders);
|
|
301649
|
+
break;
|
|
301650
|
+
}
|
|
301651
|
+
}
|
|
301652
|
+
}
|
|
301653
|
+
if (config.skipFailedRequests || config.skipSuccessfulRequests) {
|
|
301654
|
+
let decremented = false;
|
|
301655
|
+
const decrementKey = async () => {
|
|
301656
|
+
if (!decremented) {
|
|
301657
|
+
if (resetTime && Date.now() >= resetTime.getTime()) {
|
|
301658
|
+
return;
|
|
301659
|
+
}
|
|
301660
|
+
debug2("decrementing count");
|
|
301661
|
+
await config.store.decrement(key);
|
|
301662
|
+
decremented = true;
|
|
301663
|
+
}
|
|
301664
|
+
};
|
|
301665
|
+
if (config.skipFailedRequests) {
|
|
301666
|
+
if (finishPromise) {
|
|
301667
|
+
finishPromise.then(async () => {
|
|
301668
|
+
const success = await config.requestWasSuccessful(request, response);
|
|
301669
|
+
debug2("computed requestWasSuccessful as %o", success);
|
|
301670
|
+
if (!success)
|
|
301671
|
+
await decrementKey();
|
|
301672
|
+
});
|
|
301673
|
+
}
|
|
301674
|
+
if (closePromise) {
|
|
301675
|
+
closePromise.then(async () => {
|
|
301676
|
+
if (!response.writableEnded)
|
|
301677
|
+
await decrementKey();
|
|
301678
|
+
});
|
|
301679
|
+
}
|
|
301680
|
+
if (errorPromise) {
|
|
301681
|
+
errorPromise.then(async () => {
|
|
301682
|
+
await decrementKey();
|
|
301683
|
+
});
|
|
301684
|
+
}
|
|
301685
|
+
}
|
|
301686
|
+
if (config.skipSuccessfulRequests) {
|
|
301687
|
+
if (finishPromise) {
|
|
301688
|
+
finishPromise.then(async () => {
|
|
301689
|
+
const success = await config.requestWasSuccessful(request, response);
|
|
301690
|
+
debug2("computed requestWasSuccessful as %o", success);
|
|
301691
|
+
if (success)
|
|
301692
|
+
await decrementKey();
|
|
301693
|
+
});
|
|
301694
|
+
}
|
|
301695
|
+
}
|
|
301696
|
+
}
|
|
301697
|
+
if (totalHits > limit) {
|
|
301698
|
+
debug2("limit exceeded");
|
|
301699
|
+
if (config.legacyHeaders || config.standardHeaders) {
|
|
301700
|
+
debug2("set retry-after header");
|
|
301701
|
+
setRetryAfterHeader(response, info, config.windowMs);
|
|
301702
|
+
}
|
|
301703
|
+
config.handler(request, response, next, options);
|
|
301704
|
+
return;
|
|
301705
|
+
}
|
|
301706
|
+
next();
|
|
301707
|
+
});
|
|
301708
|
+
const getThrowFn = () => {
|
|
301709
|
+
throw new Error("The current store does not support the get/getKey method");
|
|
301710
|
+
};
|
|
301711
|
+
middleware.resetKey = config.store.resetKey.bind(config.store);
|
|
301712
|
+
middleware.getKey = typeof config.store.get === "function" ? config.store.get.bind(config.store) : getThrowFn;
|
|
301713
|
+
return middleware;
|
|
301714
|
+
};
|
|
301715
|
+
var rate_limit_default = rateLimit;
|
|
301716
|
+
var SECOND = 1000;
|
|
301717
|
+
var MINUTE = 60 * SECOND;
|
|
301718
|
+
var HOUR = 60 * MINUTE;
|
|
301719
|
+
var DAY = 24 * HOUR;
|
|
301720
|
+
|
|
301721
|
+
// src/rate_limit.ts
|
|
301722
|
+
var RATE_LIMIT_ENV = "PUBLISHER_RATE_LIMIT";
|
|
301723
|
+
function parseRateLimit(raw) {
|
|
301724
|
+
if (raw === undefined || raw.trim() === "")
|
|
301725
|
+
return;
|
|
301726
|
+
const n = Number(raw);
|
|
301727
|
+
if (!Number.isInteger(n) || n < 0) {
|
|
301728
|
+
throw new Error(`${RATE_LIMIT_ENV} must be a non-negative integer (requests per minute per client); got "${raw}"`);
|
|
301729
|
+
}
|
|
301730
|
+
return n === 0 ? undefined : n;
|
|
301731
|
+
}
|
|
301732
|
+
function isOperationalPath(req) {
|
|
301733
|
+
return req.path === "/metrics" || req.path.startsWith("/health");
|
|
301734
|
+
}
|
|
301735
|
+
function rateLimitMiddleware(perMinute) {
|
|
301736
|
+
const enabled = perMinute !== undefined;
|
|
301737
|
+
return rate_limit_default({
|
|
301738
|
+
windowMs: 60000,
|
|
301739
|
+
limit: perMinute ?? Number.MAX_SAFE_INTEGER,
|
|
301740
|
+
standardHeaders: "draft-7",
|
|
301741
|
+
legacyHeaders: false,
|
|
301742
|
+
skip: (req) => !enabled || isOperationalPath(req),
|
|
301743
|
+
message: {
|
|
301744
|
+
code: 429,
|
|
301745
|
+
message: `Too many requests: this client exceeded ${perMinute} requests per minute.`
|
|
301746
|
+
}
|
|
301747
|
+
});
|
|
301748
|
+
}
|
|
301749
|
+
|
|
299239
301750
|
// src/server.ts
|
|
299240
301751
|
assertSupportedNodeVersion();
|
|
299241
301752
|
function parseArgs() {
|
|
@@ -299322,6 +301833,7 @@ var isDevelopment = process.env["NODE_ENV"] === "development";
|
|
|
299322
301833
|
var app = import_express.default();
|
|
299323
301834
|
app.use(loggerMiddleware);
|
|
299324
301835
|
app.use(httpMetricsMiddleware);
|
|
301836
|
+
app.use(rateLimitMiddleware(parseRateLimit(process.env[RATE_LIMIT_ENV])));
|
|
299325
301837
|
checkHeapConfiguration();
|
|
299326
301838
|
var environmentStore = new EnvironmentStore(SERVER_ROOT);
|
|
299327
301839
|
var watchModeController = new WatchModeController(environmentStore);
|