@usherlabs/cex-broker 0.2.51 → 0.2.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/cex-canonical-orderbook-export.js +3 -3
- package/dist/commands/cex-canonical-orderbook-export.js.map +1 -1
- package/dist/commands/cli.js +216 -22
- package/dist/commands/market-data-vendor-backfill.js +3 -3
- package/dist/commands/market-data-vendor-backfill.js.map +1 -1
- package/dist/helpers/deposit-archive-poller.d.ts +33 -0
- package/dist/index.js +217 -23
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
|
@@ -7,6 +7,38 @@ export type DepositArchivePollerConfig = {
|
|
|
7
7
|
lookbackMs: number;
|
|
8
8
|
depositsLimit: number;
|
|
9
9
|
};
|
|
10
|
+
type BinanceUnlockProgressState = "pending" | "credited_not_withdrawable" | "ok" | "failed" | "unknown";
|
|
11
|
+
type BinanceUnlockProgressQuality = "valid" | "unknown" | "contradictory";
|
|
12
|
+
declare const BINANCE_UNLOCK_PROGRESS_SOURCE: {
|
|
13
|
+
readonly venue: "binance";
|
|
14
|
+
readonly endpoint: "GET /sapi/v1/capital/deposit/hisrec";
|
|
15
|
+
readonly fields: {
|
|
16
|
+
readonly status: "info.status";
|
|
17
|
+
readonly confirmTimes: "info.confirmTimes";
|
|
18
|
+
readonly unlockConfirm: "info.unlockConfirm";
|
|
19
|
+
readonly completeTime: "info.completeTime";
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
export type BinanceUnlockProgress = {
|
|
23
|
+
version: 1;
|
|
24
|
+
state: BinanceUnlockProgressState;
|
|
25
|
+
progress_state: BinanceUnlockProgressQuality;
|
|
26
|
+
reason: string | null;
|
|
27
|
+
native_status: number | null;
|
|
28
|
+
current: number | null;
|
|
29
|
+
credit_required: number | null;
|
|
30
|
+
unlock_required: number | null;
|
|
31
|
+
complete_time: number | null;
|
|
32
|
+
observed_at: string;
|
|
33
|
+
source: typeof BINANCE_UNLOCK_PROGRESS_SOURCE;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Extracts Binance's native deposit confirmation fields without trusting
|
|
37
|
+
* ccxt's normalized status. `confirmTimes` is normally the venue string
|
|
38
|
+
* `"current/credit_required"`; the object form is accepted only for callers
|
|
39
|
+
* that already decoded that exact field.
|
|
40
|
+
*/
|
|
41
|
+
export declare function parseBinanceUnlockProgress(record: Record<string, unknown>): BinanceUnlockProgress;
|
|
10
42
|
/**
|
|
11
43
|
* Advances the inclusive since-watermark only across a fully observed,
|
|
12
44
|
* terminal prefix of deposit history.
|
|
@@ -31,3 +63,4 @@ export declare class DepositArchivePoller {
|
|
|
31
63
|
stop(): Promise<void>;
|
|
32
64
|
pollAllOnce(): Promise<boolean>;
|
|
33
65
|
}
|
|
66
|
+
export {};
|
package/dist/index.js
CHANGED
|
@@ -292725,6 +292725,16 @@ function withTimeout(promise, timeoutMs, label) {
|
|
|
292725
292725
|
return Promise.race([promise, expiry]).finally(() => clearTimeout(timer));
|
|
292726
292726
|
}
|
|
292727
292727
|
var ALL_CURRENCIES_CODE = "*";
|
|
292728
|
+
var BINANCE_UNLOCK_PROGRESS_SOURCE = {
|
|
292729
|
+
venue: "binance",
|
|
292730
|
+
endpoint: "GET /sapi/v1/capital/deposit/hisrec",
|
|
292731
|
+
fields: {
|
|
292732
|
+
status: "info.status",
|
|
292733
|
+
confirmTimes: "info.confirmTimes",
|
|
292734
|
+
unlockConfirm: "info.unlockConfirm",
|
|
292735
|
+
completeTime: "info.completeTime"
|
|
292736
|
+
}
|
|
292737
|
+
};
|
|
292728
292738
|
function depositTimestamp(record) {
|
|
292729
292739
|
const observedAt = depositField(record, [
|
|
292730
292740
|
"timestamp",
|
|
@@ -292737,12 +292747,185 @@ function depositTimestamp(record) {
|
|
|
292737
292747
|
const timestamp = typeof observedAt === "number" ? observedAt : typeof observedAt === "string" ? Date.parse(observedAt) : Number.NaN;
|
|
292738
292748
|
return Number.isFinite(timestamp) ? timestamp : undefined;
|
|
292739
292749
|
}
|
|
292740
|
-
function
|
|
292741
|
-
|
|
292742
|
-
|
|
292743
|
-
|
|
292750
|
+
function parseNonNegativeInteger(value) {
|
|
292751
|
+
if (typeof value === "number") {
|
|
292752
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
292753
|
+
}
|
|
292754
|
+
if (typeof value !== "string") {
|
|
292755
|
+
return;
|
|
292756
|
+
}
|
|
292757
|
+
const trimmed = value.trim();
|
|
292758
|
+
if (!/^\d+$/.test(trimmed)) {
|
|
292759
|
+
return;
|
|
292760
|
+
}
|
|
292761
|
+
const parsed = Number(trimmed);
|
|
292762
|
+
return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined;
|
|
292763
|
+
}
|
|
292764
|
+
function parseConfirmTimes(value) {
|
|
292765
|
+
if (value === undefined || value === null) {
|
|
292766
|
+
return { reason: "missing_confirmTimes" };
|
|
292767
|
+
}
|
|
292768
|
+
let currentValue;
|
|
292769
|
+
let creditRequiredValue;
|
|
292770
|
+
if (typeof value === "string") {
|
|
292771
|
+
const match = /^(\d+)\s*\/\s*(\d+)$/.exec(value.trim());
|
|
292772
|
+
if (!match) {
|
|
292773
|
+
return { reason: "invalid_confirmTimes" };
|
|
292774
|
+
}
|
|
292775
|
+
currentValue = match[1];
|
|
292776
|
+
creditRequiredValue = match[2];
|
|
292777
|
+
} else {
|
|
292778
|
+
const record = asRecord(value);
|
|
292779
|
+
if (!record) {
|
|
292780
|
+
return { reason: "invalid_confirmTimes" };
|
|
292781
|
+
}
|
|
292782
|
+
currentValue = record.current;
|
|
292783
|
+
creditRequiredValue = record.credit_required;
|
|
292784
|
+
}
|
|
292785
|
+
const current = parseNonNegativeInteger(currentValue);
|
|
292786
|
+
const creditRequired = parseNonNegativeInteger(creditRequiredValue);
|
|
292787
|
+
if (current === undefined || creditRequired === undefined) {
|
|
292788
|
+
return { reason: "invalid_confirmTimes" };
|
|
292744
292789
|
}
|
|
292745
|
-
return
|
|
292790
|
+
return { current, creditRequired, reason: null };
|
|
292791
|
+
}
|
|
292792
|
+
function parseOptionalNonNegativeInteger(value, fieldName) {
|
|
292793
|
+
if (value === undefined || value === null) {
|
|
292794
|
+
return { value: undefined, reason: null };
|
|
292795
|
+
}
|
|
292796
|
+
const parsed = parseNonNegativeInteger(value);
|
|
292797
|
+
return parsed === undefined ? { value: undefined, reason: `invalid_${fieldName}` } : { value: parsed, reason: null };
|
|
292798
|
+
}
|
|
292799
|
+
function parseBinanceUnlockProgress(record) {
|
|
292800
|
+
const info = asRecord(record.info);
|
|
292801
|
+
const nativeStatus = parseNonNegativeInteger(info?.status);
|
|
292802
|
+
const confirmations = parseConfirmTimes(info?.confirmTimes);
|
|
292803
|
+
const unlockRequired = parseOptionalNonNegativeInteger(info?.unlockConfirm, "unlockConfirm");
|
|
292804
|
+
const completeTime = parseOptionalNonNegativeInteger(info?.completeTime, "completeTime");
|
|
292805
|
+
let quality = "valid";
|
|
292806
|
+
let reason = null;
|
|
292807
|
+
if (nativeStatus === undefined) {
|
|
292808
|
+
quality = "unknown";
|
|
292809
|
+
reason = "missing_or_invalid_status";
|
|
292810
|
+
} else if (confirmations.reason !== null) {
|
|
292811
|
+
quality = "unknown";
|
|
292812
|
+
reason = confirmations.reason;
|
|
292813
|
+
} else if (unlockRequired.reason !== null) {
|
|
292814
|
+
quality = "unknown";
|
|
292815
|
+
reason = unlockRequired.reason;
|
|
292816
|
+
} else if (completeTime.reason !== null) {
|
|
292817
|
+
quality = "unknown";
|
|
292818
|
+
reason = completeTime.reason;
|
|
292819
|
+
}
|
|
292820
|
+
let state = "unknown";
|
|
292821
|
+
switch (nativeStatus) {
|
|
292822
|
+
case 0:
|
|
292823
|
+
state = "pending";
|
|
292824
|
+
break;
|
|
292825
|
+
case 1:
|
|
292826
|
+
if (quality === "valid" && confirmations.current !== undefined && unlockRequired.value !== undefined && confirmations.current >= unlockRequired.value) {
|
|
292827
|
+
state = "ok";
|
|
292828
|
+
} else if (quality === "valid") {
|
|
292829
|
+
state = "credited_not_withdrawable";
|
|
292830
|
+
}
|
|
292831
|
+
break;
|
|
292832
|
+
case 2:
|
|
292833
|
+
case 7:
|
|
292834
|
+
state = "failed";
|
|
292835
|
+
break;
|
|
292836
|
+
case 6:
|
|
292837
|
+
state = "credited_not_withdrawable";
|
|
292838
|
+
break;
|
|
292839
|
+
case 8:
|
|
292840
|
+
state = "pending";
|
|
292841
|
+
break;
|
|
292842
|
+
default:
|
|
292843
|
+
quality = "unknown";
|
|
292844
|
+
reason = "unsupported_status";
|
|
292845
|
+
break;
|
|
292846
|
+
}
|
|
292847
|
+
return {
|
|
292848
|
+
version: 1,
|
|
292849
|
+
state,
|
|
292850
|
+
progress_state: quality,
|
|
292851
|
+
reason,
|
|
292852
|
+
native_status: nativeStatus ?? null,
|
|
292853
|
+
current: confirmations.current ?? null,
|
|
292854
|
+
credit_required: confirmations.creditRequired ?? null,
|
|
292855
|
+
unlock_required: unlockRequired.value ?? null,
|
|
292856
|
+
complete_time: completeTime.value ?? null,
|
|
292857
|
+
observed_at: new Date().toISOString(),
|
|
292858
|
+
source: BINANCE_UNLOCK_PROGRESS_SOURCE
|
|
292859
|
+
};
|
|
292860
|
+
}
|
|
292861
|
+
function unlockProgressKey(progress) {
|
|
292862
|
+
return JSON.stringify({
|
|
292863
|
+
version: progress.version,
|
|
292864
|
+
state: progress.state,
|
|
292865
|
+
progress_state: progress.progress_state,
|
|
292866
|
+
reason: progress.reason,
|
|
292867
|
+
native_status: progress.native_status,
|
|
292868
|
+
current: progress.current,
|
|
292869
|
+
credit_required: progress.credit_required,
|
|
292870
|
+
unlock_required: progress.unlock_required,
|
|
292871
|
+
complete_time: progress.complete_time
|
|
292872
|
+
});
|
|
292873
|
+
}
|
|
292874
|
+
function progressWatermark(progress, previous) {
|
|
292875
|
+
if (progress.progress_state !== "valid" || progress.current === null || progress.credit_required === null || progress.unlock_required === null) {
|
|
292876
|
+
return previous;
|
|
292877
|
+
}
|
|
292878
|
+
return {
|
|
292879
|
+
current: Math.max(previous?.current ?? 0, progress.current),
|
|
292880
|
+
credit_required: progress.credit_required,
|
|
292881
|
+
unlock_required: progress.unlock_required
|
|
292882
|
+
};
|
|
292883
|
+
}
|
|
292884
|
+
function classifyBinanceDeposit(record, previous) {
|
|
292885
|
+
let progress = parseBinanceUnlockProgress(record);
|
|
292886
|
+
let highWatermark = previous?.highWatermark;
|
|
292887
|
+
if (progress.progress_state === "valid" && progress.current !== null && progress.credit_required !== null && progress.unlock_required !== null && previous?.highWatermark !== undefined && (progress.current < previous.highWatermark.current || progress.credit_required !== previous.highWatermark.credit_required || progress.unlock_required !== previous.highWatermark.unlock_required)) {
|
|
292888
|
+
progress = {
|
|
292889
|
+
...progress,
|
|
292890
|
+
state: "unknown",
|
|
292891
|
+
progress_state: "contradictory",
|
|
292892
|
+
reason: "confirmation_progress_regressed_or_requirement_changed"
|
|
292893
|
+
};
|
|
292894
|
+
} else {
|
|
292895
|
+
highWatermark = progressWatermark(progress, previous?.highWatermark);
|
|
292896
|
+
}
|
|
292897
|
+
return {
|
|
292898
|
+
archiveStatus: progress.state,
|
|
292899
|
+
holdCursor: progress.state === "pending" || progress.state === "credited_not_withdrawable" || progress.state === "unknown",
|
|
292900
|
+
unlockProgress: progress,
|
|
292901
|
+
progressKey: unlockProgressKey(progress),
|
|
292902
|
+
highWatermark
|
|
292903
|
+
};
|
|
292904
|
+
}
|
|
292905
|
+
function classifyDeposit(exchangeId, record, previous) {
|
|
292906
|
+
if (exchangeId?.toLowerCase() === "binance") {
|
|
292907
|
+
return classifyBinanceDeposit(record, previous);
|
|
292908
|
+
}
|
|
292909
|
+
const archiveStatus = normalizeCcxtTransactionForArchive(record).status ?? "";
|
|
292910
|
+
const status = normalizeDepositStatus(depositField(record, ["status", "state"]));
|
|
292911
|
+
return {
|
|
292912
|
+
archiveStatus,
|
|
292913
|
+
holdCursor: archiveStatus === "credited_not_withdrawable" || status === "pending",
|
|
292914
|
+
progressKey: JSON.stringify({ status: archiveStatus })
|
|
292915
|
+
};
|
|
292916
|
+
}
|
|
292917
|
+
function depositIdentity(input) {
|
|
292918
|
+
if (input.externalId === undefined && input.txid === undefined) {
|
|
292919
|
+
return;
|
|
292920
|
+
}
|
|
292921
|
+
return JSON.stringify({
|
|
292922
|
+
exchange: input.exchangeId,
|
|
292923
|
+
account: input.accountSelector,
|
|
292924
|
+
coin: input.coin === undefined ? "" : String(input.coin),
|
|
292925
|
+
network: input.network === undefined ? "" : String(input.network),
|
|
292926
|
+
external_id: input.externalId ?? "",
|
|
292927
|
+
txid: input.txid ?? ""
|
|
292928
|
+
});
|
|
292746
292929
|
}
|
|
292747
292930
|
function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
|
|
292748
292931
|
if (deposits.length >= depositsLimit) {
|
|
@@ -292756,9 +292939,8 @@ function nextDepositCursor(deposits, currentSince, depositsLimit, exchangeId) {
|
|
|
292756
292939
|
return currentSince;
|
|
292757
292940
|
}
|
|
292758
292941
|
const timestamp = depositTimestamp(record);
|
|
292759
|
-
const
|
|
292760
|
-
const
|
|
292761
|
-
const isPending = archiveStatus === "credited_not_withdrawable" || status === "pending";
|
|
292942
|
+
const classification = classifyDeposit(exchangeId, record);
|
|
292943
|
+
const isPending = classification.holdCursor;
|
|
292762
292944
|
if (timestamp === undefined) {
|
|
292763
292945
|
if (isPending) {
|
|
292764
292946
|
return currentSince;
|
|
@@ -292878,7 +293060,8 @@ class DepositArchivePoller {
|
|
|
292878
293060
|
if (!record) {
|
|
292879
293061
|
continue;
|
|
292880
293062
|
}
|
|
292881
|
-
const
|
|
293063
|
+
const info = asRecord(record.info);
|
|
293064
|
+
const assetSymbol = depositField(record, ["currency", "code", "asset"]) ?? info?.coin;
|
|
292882
293065
|
const amount = depositField(record, ["amount"]);
|
|
292883
293066
|
const address = depositField(record, [
|
|
292884
293067
|
"address",
|
|
@@ -292887,8 +293070,21 @@ class DepositArchivePoller {
|
|
|
292887
293070
|
"destination"
|
|
292888
293071
|
]);
|
|
292889
293072
|
const txid = depositField(record, ["txid", "txId", "tx_hash", "txHash"]);
|
|
292890
|
-
const network = depositField(record, ["network", "chain"]);
|
|
292891
|
-
const
|
|
293073
|
+
const network = depositField(record, ["network", "chain"]) ?? info?.network;
|
|
293074
|
+
const depositTxid = txid === undefined ? undefined : String(txid);
|
|
293075
|
+
const identity = depositIdentity({
|
|
293076
|
+
exchangeId: target.exchangeId,
|
|
293077
|
+
accountSelector: target.account.label,
|
|
293078
|
+
coin: assetSymbol,
|
|
293079
|
+
network,
|
|
293080
|
+
externalId: depositTxid,
|
|
293081
|
+
txid: depositTxid
|
|
293082
|
+
});
|
|
293083
|
+
const lastArchived = identity === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(identity);
|
|
293084
|
+
const classification = classifyDeposit(target.exchangeId, record, lastArchived);
|
|
293085
|
+
if (lastArchived && lastArchived.status === classification.archiveStatus && lastArchived.progressKey === classification.progressKey) {
|
|
293086
|
+
continue;
|
|
293087
|
+
}
|
|
292892
293088
|
const creditedAt = depositField(record, [
|
|
292893
293089
|
"creditedAt",
|
|
292894
293090
|
"credited_at",
|
|
@@ -292897,11 +293093,7 @@ class DepositArchivePoller {
|
|
|
292897
293093
|
"timestamp",
|
|
292898
293094
|
"datetime"
|
|
292899
293095
|
]);
|
|
292900
|
-
const
|
|
292901
|
-
const lastArchived = depositTxid === undefined ? undefined : this.#lastArchivedByTarget.get(key)?.get(depositTxid);
|
|
292902
|
-
if (lastArchived && lastArchived.status === archiveStatus) {
|
|
292903
|
-
continue;
|
|
292904
|
-
}
|
|
293096
|
+
const payload = classification.unlockProgress === undefined ? record : { ...record, unlock_progress: classification.unlockProgress };
|
|
292905
293097
|
this.params.archiver.enqueue(buildTransferEventArchiveRow({
|
|
292906
293098
|
tags: buildCommonArchiveTags({
|
|
292907
293099
|
deploymentId: this.params.archiver.getDeploymentId(),
|
|
@@ -292912,25 +293104,27 @@ class DepositArchivePoller {
|
|
|
292912
293104
|
transfer: {
|
|
292913
293105
|
eventKind: "deposit",
|
|
292914
293106
|
lifecycleAction: "observe_deposit",
|
|
292915
|
-
status: archiveStatus,
|
|
293107
|
+
status: classification.archiveStatus,
|
|
292916
293108
|
amount: amount === undefined ? undefined : String(amount),
|
|
292917
293109
|
address: address === undefined ? undefined : String(address),
|
|
292918
293110
|
network: network === undefined ? undefined : String(network),
|
|
292919
293111
|
externalId: depositTxid,
|
|
292920
293112
|
txid: depositTxid,
|
|
292921
293113
|
exchangeTimestamp: normalizeTimestamp(creditedAt),
|
|
292922
|
-
payload
|
|
293114
|
+
payload
|
|
292923
293115
|
}
|
|
292924
293116
|
}));
|
|
292925
|
-
if (
|
|
293117
|
+
if (identity !== undefined) {
|
|
292926
293118
|
let targetDeposits2 = this.#lastArchivedByTarget.get(key);
|
|
292927
293119
|
if (!targetDeposits2) {
|
|
292928
293120
|
targetDeposits2 = new Map;
|
|
292929
293121
|
this.#lastArchivedByTarget.set(key, targetDeposits2);
|
|
292930
293122
|
}
|
|
292931
|
-
targetDeposits2.set(
|
|
292932
|
-
status: archiveStatus,
|
|
292933
|
-
timestamp: depositTimestamp(record)
|
|
293123
|
+
targetDeposits2.set(identity, {
|
|
293124
|
+
status: classification.archiveStatus,
|
|
293125
|
+
timestamp: depositTimestamp(record),
|
|
293126
|
+
progressKey: classification.progressKey,
|
|
293127
|
+
highWatermark: classification.highWatermark
|
|
292934
293128
|
});
|
|
292935
293129
|
}
|
|
292936
293130
|
archived += 1;
|
|
@@ -313345,4 +313539,4 @@ export {
|
|
|
313345
313539
|
CEXBroker as default
|
|
313346
313540
|
};
|
|
313347
313541
|
|
|
313348
|
-
//# debugId=
|
|
313542
|
+
//# debugId=72C8C3614857C3F664756E2164756E21
|