envio 3.15.0 → 3.16.0-subgraph
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -34
- package/evm.schema.json +11 -2
- package/fuel.schema.json +10 -1
- package/index.d.ts +6 -0
- package/licenses/README.md +1 -1
- package/package.json +10 -9
- package/src/Batch.res.mjs +5 -5
- package/src/BatchProcessing.res +1 -0
- package/src/BatchProcessing.res.mjs +1 -0
- package/src/Bin.res +7 -0
- package/src/Bin.res.mjs +8 -0
- package/src/ChainFetching.res +450 -16
- package/src/ChainFetching.res.mjs +403 -32
- package/src/ChainState.res +124 -7
- package/src/ChainState.res.mjs +128 -67
- package/src/ChainState.resi +26 -0
- package/src/Clock.res +57 -0
- package/src/Clock.res.mjs +72 -0
- package/src/Config.res +31 -3
- package/src/Config.res.mjs +15 -4
- package/src/ContractRegisterContext.res +3 -6
- package/src/ContractRegisterContext.res.mjs +4 -2
- package/src/Core.res +95 -36
- package/src/Core.res.mjs +90 -1
- package/src/CrossChainState.res +12 -2
- package/src/CrossChainState.res.mjs +3 -3
- package/src/Ecosystem.res +7 -4
- package/src/Ecosystem.res.mjs +8 -8
- package/src/EffectCacheLookup.res +98 -0
- package/src/EffectCacheLookup.res.mjs +116 -0
- package/src/EffectState.res +10 -2
- package/src/EffectState.res.mjs +8 -4
- package/src/EffectState.resi +3 -1
- package/src/EntityId.res +2 -7
- package/src/EntityId.res.mjs +1 -8
- package/src/EntityTables.res +4 -1
- package/src/EntityTables.res.mjs +1 -1
- package/src/Env.res +20 -0
- package/src/Env.res.mjs +16 -0
- package/src/EventProcessing.res +38 -1
- package/src/EventProcessing.res.mjs +28 -1
- package/src/FetchState.res +408 -73
- package/src/FetchState.res.mjs +361 -91
- package/src/FinalizeBackfill.res +2 -4
- package/src/FinalizeBackfill.res.mjs +3 -4
- package/src/HandlerLoader.res +39 -15
- package/src/HandlerLoader.res.mjs +27 -8
- package/src/HandlerRegister.res +4 -0
- package/src/HandlerRegister.res.mjs +5 -2
- package/src/HandlerRegister.resi +2 -0
- package/src/InMemoryTable.res +17 -16
- package/src/InMemoryTable.res.mjs +14 -15
- package/src/IndexerState.res +28 -4
- package/src/IndexerState.res.mjs +19 -6
- package/src/IndexerState.resi +4 -0
- package/src/Internal.res +61 -49
- package/src/Internal.res.mjs +36 -80
- package/src/LoadLayer.res +54 -15
- package/src/LoadLayer.res.mjs +75 -45
- package/src/LoadLayer.resi +26 -0
- package/src/Main.res +14 -1
- package/src/Main.res.mjs +4 -3
- package/src/Persistence.res +3 -23
- package/src/Persistence.res.mjs +3 -3
- package/src/PreloadLookahead.res +202 -0
- package/src/PreloadLookahead.res.mjs +218 -0
- package/src/Rollback.res +0 -1
- package/src/Rollback.res.mjs +1 -1
- package/src/RunScope.res +3 -0
- package/src/SafeCheckpointTracking.res +4 -4
- package/src/SafeCheckpointTracking.res.mjs +1 -1
- package/src/SimulateItems.res +42 -26
- package/src/SimulateItems.res.mjs +40 -16
- package/src/StagedRows.res +106 -0
- package/src/StagedRows.res.mjs +102 -0
- package/src/Staging.res +32 -14
- package/src/Staging.res.mjs +17 -20
- package/src/Storage.res +303 -948
- package/src/Storage.res.mjs +229 -708
- package/src/SubgraphLog.res +19 -0
- package/src/SubgraphLog.res.mjs +26 -0
- package/src/Supervisor.res +75 -56
- package/src/Supervisor.res.mjs +55 -26
- package/src/SyncWrites.res +107 -0
- package/src/SyncWrites.res.mjs +98 -0
- package/src/TestIndexer.res +22 -27
- package/src/TestIndexer.res.mjs +20 -17
- package/src/Throttler.res +15 -8
- package/src/Throttler.res.mjs +9 -6
- package/src/Throttler.resi +1 -1
- package/src/UserContext.res +419 -46
- package/src/UserContext.res.mjs +318 -30
- package/src/Utils.res +17 -6
- package/src/Utils.res.mjs +12 -6
- package/src/Writing.res +0 -1
- package/src/Writing.res.mjs +1 -1
- package/src/bindings/ClickHouse.res +0 -322
- package/src/bindings/ClickHouse.res.mjs +0 -338
- package/src/bindings/StorageClient.res +108 -174
- package/src/bindings/StorageClient.res.mjs +19 -8
- package/src/bindings/Vitest.res +0 -26
- package/src/bindings/Vitest.res.mjs +0 -3
- package/src/db/Table.res +24 -150
- package/src/db/Table.res.mjs +38 -272
- package/src/sources/AddressSet.res +2 -0
- package/src/sources/AddressStore.res +3 -0
- package/src/sources/CallSource.res +274 -0
- package/src/sources/CallSource.res.mjs +196 -0
- package/src/sources/ChainSources.res +2 -0
- package/src/sources/ChainSources.res.mjs +2 -2
- package/src/sources/EvmChain.res +10 -1
- package/src/sources/EvmChain.res.mjs +6 -3
- package/src/sources/EvmHyperSyncSource.res +2 -2
- package/src/sources/EvmHyperSyncSource.res.mjs +1 -1
- package/src/sources/EvmRpcClient.res +13 -3
- package/src/sources/EvmRpcClient.res.mjs +4 -3
- package/src/sources/EvmRpcWs.res +4 -3
- package/src/sources/EvmRpcWs.res.mjs +2 -2
- package/src/sources/HeightFeed.res +31 -10
- package/src/sources/HeightFeed.res.mjs +10 -6
- package/src/sources/HeightStream.res +32 -17
- package/src/sources/HeightStream.res.mjs +10 -10
- package/src/sources/HyperSync.res +2 -1
- package/src/sources/HyperSync.res.mjs +2 -1
- package/src/sources/HyperSyncSSE.res +4 -6
- package/src/sources/HyperSyncSSE.res.mjs +2 -2
- package/src/sources/RpcSource.res +10 -1
- package/src/sources/RpcSource.res.mjs +4 -3
- package/src/sources/Source.res +3 -0
- package/src/sources/SourceManager.res +84 -77
- package/src/sources/SourceManager.res.mjs +17 -20
- package/src/sources/SourceManager.resi +3 -4
- package/src/sources/StartBlockResolver.res +1 -0
- package/src/sources/StartBlockResolver.res.mjs +3 -2
- package/src/sources/SvmHyperSyncSource.res +2 -2
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
- package/src/subgraph/abi-types.ts +84 -0
- package/src/subgraph/aggregations.ts +745 -0
- package/src/subgraph/as-package.ts +143 -0
- package/src/subgraph/assemblyscript-regex.ts +67 -0
- package/src/subgraph/assemblyscript.ts +731 -0
- package/src/subgraph/blocks.ts +398 -0
- package/src/subgraph/calls-source.ts +702 -0
- package/src/subgraph/calls.ts +258 -0
- package/src/subgraph/conformance.ts +106 -0
- package/src/subgraph/errors.ts +85 -0
- package/src/subgraph/graft.ts +73 -0
- package/src/subgraph/graph-cli-worker.mjs +35 -0
- package/src/subgraph/graph-cli.ts +900 -0
- package/src/subgraph/graph-node-decimal.ts +199 -0
- package/src/subgraph/graph-ts-enums.ts +28 -0
- package/src/subgraph/graph-ts-types/VERSION +2 -0
- package/src/subgraph/graph-ts-types/chain/arweave.d.ts +70 -0
- package/src/subgraph/graph-ts-types/chain/cosmos.d.ts +327 -0
- package/src/subgraph/graph-ts-types/chain/ethereum.d.ts +233 -0
- package/src/subgraph/graph-ts-types/chain/near.d.ts +253 -0
- package/src/subgraph/graph-ts-types/chain/starknet.d.ts +32 -0
- package/src/subgraph/graph-ts-types/common/collections.d.ts +136 -0
- package/src/subgraph/graph-ts-types/common/conversion.d.ts +11 -0
- package/src/subgraph/graph-ts-types/common/datasource.d.ts +30 -0
- package/src/subgraph/graph-ts-types/common/eager-offset.d.ts +0 -0
- package/src/subgraph/graph-ts-types/common/json.d.ts +17 -0
- package/src/subgraph/graph-ts-types/common/numbers.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/value.d.ts +120 -0
- package/src/subgraph/graph-ts-types/common/yaml.d.ts +90 -0
- package/src/subgraph/graph-ts-types/global/global.d.ts +194 -0
- package/src/subgraph/graph-ts-types/helper-functions.d.ts +22 -0
- package/src/subgraph/graph-ts-types/index.d.ts +102 -0
- package/src/subgraph/graph-ts.ts +4339 -0
- package/src/subgraph/hosts.ts +554 -0
- package/src/subgraph/log.ts +28 -0
- package/src/subgraph/receipts.ts +184 -0
- package/src/subgraph/register-slots.ts +93 -0
- package/src/subgraph/rollup.ts +109 -0
- package/src/subgraph/rpc.ts +337 -0
- package/src/subgraph/runtime.ts +2059 -0
- package/src/subgraph/scope.ts +176 -0
- package/src/subgraph/stats.ts +42 -0
- package/src/subgraph/template-instances.ts +66 -0
- package/src/subgraph/toolchain.ts +327 -0
- package/src/subgraph/transient.ts +176 -0
- package/svm.schema.json +11 -2
- package/src/PgWriting.res +0 -36
- package/src/PgWriting.res.mjs +0 -34
- package/src/db/IndexDefinition.res +0 -47
- package/src/db/IndexDefinition.res.mjs +0 -67
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import type { AggregationDef } from "./aggregations.ts";
|
|
2
|
+
import type { RegisterSlot } from "./register-slots.ts";
|
|
3
|
+
|
|
4
|
+
/** What a wrapper is doing when it runs the mapping. */
|
|
5
|
+
export type Mode = "handler" | "register";
|
|
6
|
+
|
|
7
|
+
export type SubgraphSchema = {
|
|
8
|
+
timestampFields: Record<string, string[]>;
|
|
9
|
+
entityListFields: Record<string, string[]>;
|
|
10
|
+
entityFields: Record<string, string[]>;
|
|
11
|
+
entityRefFields: Record<string, string[]>;
|
|
12
|
+
/** Each field as the subgraph schema declares it, which is what a mapping reads. */
|
|
13
|
+
entityFieldTypes: Record<
|
|
14
|
+
string,
|
|
15
|
+
Record<
|
|
16
|
+
string,
|
|
17
|
+
{
|
|
18
|
+
kind: string;
|
|
19
|
+
target?: string;
|
|
20
|
+
derivedFrom?: string;
|
|
21
|
+
list: boolean;
|
|
22
|
+
required?: boolean;
|
|
23
|
+
/** `[String]`: stored as a list of non-null elements. */
|
|
24
|
+
nullableElements?: boolean;
|
|
25
|
+
}
|
|
26
|
+
>
|
|
27
|
+
>;
|
|
28
|
+
/** The context accessor envio generated for each entity, by schema name. */
|
|
29
|
+
entityAccessors: Record<string, string>;
|
|
30
|
+
/** `@entity(timeseries: true)` types: saved with an automatic id and timestamp. */
|
|
31
|
+
timeseriesEntities: string[];
|
|
32
|
+
/** `@entity(immutable: true, skipDuplicates: true)` types: an id stored in an earlier block keeps its first write. */
|
|
33
|
+
skipDuplicateEntities: string[];
|
|
34
|
+
aggregations: AggregationDef[];
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** One event's register pass while it runs. */
|
|
38
|
+
export type RegisterPass = {
|
|
39
|
+
chainId: number;
|
|
40
|
+
position: number;
|
|
41
|
+
done: Promise<void>;
|
|
42
|
+
settled: boolean;
|
|
43
|
+
/** The register-store keys it missed, so later passes can wait on it. */
|
|
44
|
+
missed: string[];
|
|
45
|
+
/** A read missed while an earlier pass of its chain was still running. */
|
|
46
|
+
provisional: boolean;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export type Scope = {
|
|
50
|
+
/** envio's handler or contractRegister context. */
|
|
51
|
+
context: any;
|
|
52
|
+
event: any;
|
|
53
|
+
mode: Mode;
|
|
54
|
+
schema: SubgraphSchema;
|
|
55
|
+
dataSource: {
|
|
56
|
+
name: string;
|
|
57
|
+
address: string;
|
|
58
|
+
chainId: number;
|
|
59
|
+
network: string;
|
|
60
|
+
/** The manifest's `context` entries, as `dataSource.context()` serves them. */
|
|
61
|
+
context: Record<string, { type: string; data: string }>;
|
|
62
|
+
/** A static data source without `source.address`: `address` is the emitter's, `dataSource.address()` none. */
|
|
63
|
+
wildcard?: boolean;
|
|
64
|
+
};
|
|
65
|
+
/** Addresses already registered this round, so replays stay idempotent. */
|
|
66
|
+
registered: Set<string>;
|
|
67
|
+
/**
|
|
68
|
+
* Host results the register pass has resolved, keyed by effect and input.
|
|
69
|
+
* The register pass has no envio context to run effects through, so it
|
|
70
|
+
* replays the mapping itself and answers from here (§5, register mode).
|
|
71
|
+
*/
|
|
72
|
+
resolved?: Map<string, { value?: unknown; error?: unknown }>;
|
|
73
|
+
/** Lookups the current register round is waiting on. */
|
|
74
|
+
awaiting?: Promise<unknown>[];
|
|
75
|
+
/** The block a contract call is evaluated against, as graph-node does. */
|
|
76
|
+
blockNumber: number;
|
|
77
|
+
/** The block's time in seconds: what a timeseries entity is stamped with. */
|
|
78
|
+
blockSeconds?: () => number | string;
|
|
79
|
+
/** Timeseries entities saved so far by this run of the handler. */
|
|
80
|
+
timeseriesSequence?: number;
|
|
81
|
+
/** Where this run's timeseries ids start in its block's id sequence. */
|
|
82
|
+
timeseriesBase?: number;
|
|
83
|
+
/** Where the event sits in its chain, for what the register pass may read. */
|
|
84
|
+
position?: number;
|
|
85
|
+
/** The register round's own writes, kept only once the round completes. */
|
|
86
|
+
writes?: Map<string, Record<string, unknown> | null>;
|
|
87
|
+
/** This register pass, as other passes of its batch wait on it. */
|
|
88
|
+
pass?: RegisterPass;
|
|
89
|
+
/** Entities this pass already waited on another pass for: it waits once. */
|
|
90
|
+
waitedKeys?: Set<string>;
|
|
91
|
+
/** The register pass's slot (`register-slots.ts`), held while it runs. */
|
|
92
|
+
slot?: RegisterSlot;
|
|
93
|
+
/** The round's wait includes an earlier pass: the slot is given back for it. */
|
|
94
|
+
waitsOnPass?: boolean;
|
|
95
|
+
/** The mapping module, so `ipfs.map` can reach the callback it names. */
|
|
96
|
+
mappingExports: Record<string, any>;
|
|
97
|
+
/** Set when the scope is a file data source's handler. */
|
|
98
|
+
file?: FileScope;
|
|
99
|
+
/** The block handler this run is, when it has no register pass of its own. */
|
|
100
|
+
blockHandler?: string;
|
|
101
|
+
/** File templates this run created, keyed to dedupe a replayed mapping. */
|
|
102
|
+
fileCreates?: Map<string, FileCreate>;
|
|
103
|
+
/** Register mode: instances this pass created of templates with a `once` handler. */
|
|
104
|
+
templateOnces?: { template: string; address: string }[];
|
|
105
|
+
};
|
|
106
|
+
|
|
107
|
+
/** A `DataSourceTemplate.create` of a file template, waiting for its file. */
|
|
108
|
+
export type FileCreate = {
|
|
109
|
+
template: string;
|
|
110
|
+
/** The CID or Arweave transaction id the mapping passed. */
|
|
111
|
+
source: string;
|
|
112
|
+
/** The `DataSourceContext` of `createWithContext`, if there was one. */
|
|
113
|
+
context?: unknown;
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
/** What distinguishes a file data source's handler run from a chain one. */
|
|
117
|
+
export type FileScope = {
|
|
118
|
+
kind: "ipfs" | "arweave";
|
|
119
|
+
/**
|
|
120
|
+
* The manifest's `entities` list. graph-node lets a file data source touch
|
|
121
|
+
* only these types, and refuses any other, read or write.
|
|
122
|
+
*/
|
|
123
|
+
entities: Set<string>;
|
|
124
|
+
/**
|
|
125
|
+
* What this run has written, by entity type and id. graph-node keys an entity
|
|
126
|
+
* by its causality region too, so a file data source can't see a chain
|
|
127
|
+
* entity (or another file's) even of the same type and id: a read answers
|
|
128
|
+
* from here only.
|
|
129
|
+
*/
|
|
130
|
+
written: Set<string>;
|
|
131
|
+
context?: unknown;
|
|
132
|
+
};
|
|
133
|
+
|
|
134
|
+
// A mapping runs synchronously from start to end of a round, so the scope is a
|
|
135
|
+
// plain variable rather than an AsyncLocalStorage: in this Node, a live
|
|
136
|
+
// AsyncLocalStorage installs promise hooks that tax every promise the indexer
|
|
137
|
+
// makes (~4% of indexing CPU).
|
|
138
|
+
let current: Scope | undefined;
|
|
139
|
+
|
|
140
|
+
export function runInScope<T>(scope: Scope, fn: () => T): T {
|
|
141
|
+
const previous = current;
|
|
142
|
+
current = scope;
|
|
143
|
+
try {
|
|
144
|
+
return fn();
|
|
145
|
+
} finally {
|
|
146
|
+
current = previous;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* A mapping module's top level runs during its import, after `import()` has
|
|
152
|
+
* returned. Mappings are imported one at a time, so the scope is held until
|
|
153
|
+
* the import settles.
|
|
154
|
+
*/
|
|
155
|
+
export async function runInScopeUntilSettled<T>(scope: Scope, fn: () => Promise<T>): Promise<T> {
|
|
156
|
+
const previous = current;
|
|
157
|
+
current = scope;
|
|
158
|
+
try {
|
|
159
|
+
return await fn();
|
|
160
|
+
} finally {
|
|
161
|
+
current = previous;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function currentScope(): Scope {
|
|
166
|
+
const scope = current;
|
|
167
|
+
if (!scope) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
"A graph-ts API was called outside of a mapping handler. Envio Subgraph " +
|
|
170
|
+
"runs mappings inside a scope that carries the event and the store; " +
|
|
171
|
+
"calling into graph-ts from module top-level or from a detached " +
|
|
172
|
+
"callback escapes it.",
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
return scope;
|
|
176
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `ENVIO_SUBGRAPH_SYNC_STATS=1` counts how often the sync bridge replays a
|
|
3
|
+
* mapping (§5), per phase, and prints the totals as one `SGSTATS {json}` line
|
|
4
|
+
* on stderr every 30 s and at exit. A preload round that suspends is the
|
|
5
|
+
* batching working; an execute round that suspends is a database round trip
|
|
6
|
+
* the batch waits on, one event at a time.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const enabled = process.env.ENVIO_SUBGRAPH_SYNC_STATS === "1";
|
|
10
|
+
|
|
11
|
+
type PhaseStats = { runs: number; rounds: number; maxRounds: number; byRounds: Record<string, number> };
|
|
12
|
+
const makePhase = (): PhaseStats => ({ runs: 0, rounds: 0, maxRounds: 0, byRounds: {} });
|
|
13
|
+
const stats = { preload: makePhase(), execute: makePhase() };
|
|
14
|
+
|
|
15
|
+
function record(phase: PhaseStats, rounds: number) {
|
|
16
|
+
phase.runs++;
|
|
17
|
+
phase.rounds += rounds;
|
|
18
|
+
if (rounds > phase.maxRounds) phase.maxRounds = rounds;
|
|
19
|
+
const bucket = rounds >= 5 ? "5+" : String(rounds);
|
|
20
|
+
phase.byRounds[bucket] = (phase.byRounds[bucket] ?? 0) + 1;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
if (enabled) {
|
|
24
|
+
const print = () => process.stderr.write(`SGSTATS ${JSON.stringify(stats)}\n`);
|
|
25
|
+
setInterval(print, 30_000).unref();
|
|
26
|
+
process.on("exit", print);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** `context.runSync(body)`, counting the rounds it took when stats are on. */
|
|
30
|
+
export async function runSyncCounted(context: any, body: () => void): Promise<void> {
|
|
31
|
+
if (!enabled) return context.runSync(body);
|
|
32
|
+
let rounds = 0;
|
|
33
|
+
const isPreload = context.isPreload;
|
|
34
|
+
try {
|
|
35
|
+
await context.runSync(() => {
|
|
36
|
+
rounds++;
|
|
37
|
+
body();
|
|
38
|
+
});
|
|
39
|
+
} finally {
|
|
40
|
+
record(isPreload ? stats.preload : stats.execute, rounds);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
type Instance = { address: string; block: number };
|
|
5
|
+
|
|
6
|
+
const instances = new Map<string, Instance[]>();
|
|
7
|
+
let loaded = false;
|
|
8
|
+
|
|
9
|
+
function file(): string {
|
|
10
|
+
return path.join(process.cwd(), ".envio", "subgraph-template-instances.jsonl");
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const keyOf = (chainId: number, template: string) => `${chainId}:${template}`;
|
|
14
|
+
|
|
15
|
+
function add(key: string, address: string, block: number): boolean {
|
|
16
|
+
const list = instances.get(key) ?? [];
|
|
17
|
+
const known = list.find((instance) => instance.address === address);
|
|
18
|
+
if (known) {
|
|
19
|
+
if (known.block === block) return false;
|
|
20
|
+
known.block = block;
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
list.push({ address, block });
|
|
24
|
+
instances.set(key, list);
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function load() {
|
|
29
|
+
if (loaded) return;
|
|
30
|
+
loaded = true;
|
|
31
|
+
try {
|
|
32
|
+
for (const line of readFileSync(file(), "utf8").split("\n")) {
|
|
33
|
+
if (!line) continue;
|
|
34
|
+
const { key, address, block } = JSON.parse(line);
|
|
35
|
+
add(key, address, block);
|
|
36
|
+
}
|
|
37
|
+
} catch {
|
|
38
|
+
// none saved yet
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* A template instance and the block it was created in, which its block
|
|
44
|
+
* handlers run from. envio keeps the address of a created data source and not
|
|
45
|
+
* the block graph-node starts its triggers at, so the block is kept here, in
|
|
46
|
+
* the project's `.envio` directory too: a restart loads the addresses from
|
|
47
|
+
* storage and no register pass runs for the old creations.
|
|
48
|
+
*/
|
|
49
|
+
export function rememberInstance(chainId: number, template: string, address: string, block: number) {
|
|
50
|
+
load();
|
|
51
|
+
const key = keyOf(chainId, template);
|
|
52
|
+
const lower = address.toLowerCase();
|
|
53
|
+
if (!add(key, lower, block)) return;
|
|
54
|
+
try {
|
|
55
|
+
mkdirSync(path.dirname(file()), { recursive: true });
|
|
56
|
+
appendFileSync(file(), JSON.stringify({ key, address: lower, block }) + "\n");
|
|
57
|
+
} catch {
|
|
58
|
+
// in memory still serves this run
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The instances of `template` created at or before `block`, with their creation blocks. */
|
|
63
|
+
export function instancesAt(chainId: number, template: string, block: number): Instance[] {
|
|
64
|
+
load();
|
|
65
|
+
return (instances.get(keyOf(chainId, template)) ?? []).filter((instance) => instance.block <= block);
|
|
66
|
+
}
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The pieces of a subgraph's toolchain envio fetches itself when the project's
|
|
3
|
+
* own `node_modules` can't supply them.
|
|
4
|
+
*
|
|
5
|
+
* graph-node never installs a repo's npm dependencies at index time, it runs
|
|
6
|
+
* compiled WASM. A team moving a subgraph expects one command to work in the
|
|
7
|
+
* repo as it is, and a repo whose full install fails (a dead git dependency in
|
|
8
|
+
* an old graph-cli lockfile, a vanished gluegun fork, ETARGET) would otherwise
|
|
9
|
+
* stop before the first line of a mapping ran. Only two things are needed:
|
|
10
|
+
*
|
|
11
|
+
* - the packages the mappings actually import (`@openzeppelin/subgraphs`,
|
|
12
|
+
* `@protofire/subgraph-toolkit`, ...), found by walking the mapping import
|
|
13
|
+
* graph, at the versions the repo's lockfile or package.json names;
|
|
14
|
+
* - a graph-cli, to run `graph codegen`, at a version that installs.
|
|
15
|
+
*
|
|
16
|
+
* Both are installed with npm into a cache directory of envio's own, away from
|
|
17
|
+
* the repo, with no lifecycle scripts, skipping git, file and workspace
|
|
18
|
+
* dependencies and every devDependency the mappings don't import.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { spawn } from "node:child_process";
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
24
|
+
import { builtinModules } from "node:module";
|
|
25
|
+
import { homedir } from "node:os";
|
|
26
|
+
import path from "node:path";
|
|
27
|
+
import { info, warn } from "./log.ts";
|
|
28
|
+
|
|
29
|
+
/** Where fetched packages live: `ENVIO_SUBGRAPH_CACHE_DIR`, else the user's cache. */
|
|
30
|
+
export function cacheDir(): string {
|
|
31
|
+
return (
|
|
32
|
+
process.env.ENVIO_SUBGRAPH_CACHE_DIR ||
|
|
33
|
+
path.join(process.env.XDG_CACHE_HOME || path.join(homedir(), ".cache"), "envio", "subgraph")
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* npm into `dir` (a prefix with its own node_modules), atomically: installed
|
|
39
|
+
* beside it, then renamed, so an interrupted install never leaves a half
|
|
40
|
+
* cache that looks complete.
|
|
41
|
+
*/
|
|
42
|
+
async function npmInstallInto(dir: string, specs: string[]): Promise<void> {
|
|
43
|
+
if (existsSync(path.join(dir, ".envio-installed"))) return;
|
|
44
|
+
const staging = `${dir}.tmp-${process.pid}`;
|
|
45
|
+
rmSync(staging, { recursive: true, force: true });
|
|
46
|
+
mkdirSync(staging, { recursive: true });
|
|
47
|
+
writeFileSync(path.join(staging, "package.json"), JSON.stringify({ name: "envio-subgraph-cache", private: true }));
|
|
48
|
+
const output: string[] = [];
|
|
49
|
+
await new Promise<void>((resolve, reject) => {
|
|
50
|
+
const child = spawn(
|
|
51
|
+
"npm",
|
|
52
|
+
[
|
|
53
|
+
"install",
|
|
54
|
+
"--no-audit",
|
|
55
|
+
"--no-fund",
|
|
56
|
+
"--no-package-lock",
|
|
57
|
+
"--ignore-scripts",
|
|
58
|
+
"--legacy-peer-deps",
|
|
59
|
+
"--loglevel=error",
|
|
60
|
+
...specs,
|
|
61
|
+
],
|
|
62
|
+
{ cwd: staging, env: process.env },
|
|
63
|
+
);
|
|
64
|
+
child.stdout.on("data", (chunk) => output.push(String(chunk)));
|
|
65
|
+
child.stderr.on("data", (chunk) => output.push(String(chunk)));
|
|
66
|
+
child.on("error", reject);
|
|
67
|
+
child.on("exit", (code) =>
|
|
68
|
+
code === 0 ? resolve() : reject(new Error(`npm install ${specs.join(" ")} exited with ${code}\n${output.join("").slice(-1500)}`)),
|
|
69
|
+
);
|
|
70
|
+
}).catch((error) => {
|
|
71
|
+
rmSync(staging, { recursive: true, force: true });
|
|
72
|
+
throw error;
|
|
73
|
+
});
|
|
74
|
+
writeFileSync(path.join(staging, ".envio-installed"), specs.join("\n"));
|
|
75
|
+
rmSync(dir, { recursive: true, force: true });
|
|
76
|
+
mkdirSync(path.dirname(dir), { recursive: true });
|
|
77
|
+
renameSync(staging, dir);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// --- what the repo declares -------------------------------------------------
|
|
81
|
+
|
|
82
|
+
const EXACT_VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
|
|
83
|
+
|
|
84
|
+
/** A package.json spec npm can fetch from the registry: a version or range, not a git, file, link or workspace one. */
|
|
85
|
+
function registrySpec(spec: unknown): string | null {
|
|
86
|
+
if (typeof spec !== "string" || spec === "" || spec === "*" || spec === "latest") return null;
|
|
87
|
+
if (/^(?:git|github|file|link|workspace|portal|patch|http|https|npm:|[^@\s]+\/[^@\s]+$)/.test(spec) || spec.includes("://")) {
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
return spec;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function readJson(file: string): any {
|
|
94
|
+
try {
|
|
95
|
+
return JSON.parse(readFileSync(file, "utf8"));
|
|
96
|
+
} catch {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function escapeRegExp(text: string): string {
|
|
102
|
+
return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** The version a lockfile in `dir` resolved `name` to: npm, yarn (1 and berry) or pnpm. */
|
|
106
|
+
function lockedVersion(dir: string, name: string): string | null {
|
|
107
|
+
const npmLock = readJson(path.join(dir, "package-lock.json"));
|
|
108
|
+
if (npmLock) {
|
|
109
|
+
const version = npmLock.packages?.[`node_modules/${name}`]?.version ?? npmLock.dependencies?.[name]?.version;
|
|
110
|
+
if (EXACT_VERSION.test(version ?? "")) return version;
|
|
111
|
+
}
|
|
112
|
+
const read = (file: string) => (existsSync(file) ? readFileSync(file, "utf8") : null);
|
|
113
|
+
const yarn = read(path.join(dir, "yarn.lock"));
|
|
114
|
+
if (yarn) {
|
|
115
|
+
for (const block of yarn.split(/\n\s*\n/)) {
|
|
116
|
+
const header = block.split("\n").find((line) => line !== "" && !line.startsWith("#"));
|
|
117
|
+
if (!header) continue;
|
|
118
|
+
const entries = header.replace(/:\s*$/, "").split(/,\s*/).map((entry) => entry.trim().replace(/^"|"$/g, ""));
|
|
119
|
+
if (!entries.some((entry) => entry.startsWith(`${name}@`))) continue;
|
|
120
|
+
const version = block.match(/^\s+version:?\s+"?([^"\s]+)"?/m)?.[1];
|
|
121
|
+
if (version && EXACT_VERSION.test(version)) return version;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
const pnpm = read(path.join(dir, "pnpm-lock.yaml"));
|
|
125
|
+
if (pnpm) {
|
|
126
|
+
const version = pnpm.match(new RegExp(`^\\s+'?/?${escapeRegExp(name)}@(\\d[^:(')\\s]*)`, "m"))?.[1];
|
|
127
|
+
if (version && EXACT_VERSION.test(version)) return version;
|
|
128
|
+
}
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The version of `name` to fetch: the lockfile's, else package.json's range,
|
|
134
|
+
* from the project's directory up through the workspace root; null for latest.
|
|
135
|
+
*/
|
|
136
|
+
export function declaredVersion(root: string, name: string): string | null {
|
|
137
|
+
const directories: string[] = [];
|
|
138
|
+
for (let dir = path.resolve(root); ; dir = path.dirname(dir)) {
|
|
139
|
+
directories.push(dir);
|
|
140
|
+
if (path.dirname(dir) === dir) break;
|
|
141
|
+
}
|
|
142
|
+
for (const dir of directories) {
|
|
143
|
+
const locked = lockedVersion(dir, name);
|
|
144
|
+
if (locked) return locked;
|
|
145
|
+
}
|
|
146
|
+
for (const dir of directories) {
|
|
147
|
+
const manifest = readJson(path.join(dir, "package.json"));
|
|
148
|
+
const spec = registrySpec(
|
|
149
|
+
manifest?.dependencies?.[name] ??
|
|
150
|
+
manifest?.devDependencies?.[name] ??
|
|
151
|
+
manifest?.peerDependencies?.[name] ??
|
|
152
|
+
manifest?.optionalDependencies?.[name],
|
|
153
|
+
);
|
|
154
|
+
if (spec) return spec;
|
|
155
|
+
}
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// --- the packages the mappings import -----------------------------------------
|
|
160
|
+
|
|
161
|
+
const BARE_IMPORT = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)["']([@\w][^"'\s]*)["']/g;
|
|
162
|
+
/** A name npm would accept: lowercase, optionally scoped. */
|
|
163
|
+
const NPM_NAME = /^(?:@[a-z0-9~-][a-z0-9._~-]*\/)?[a-z0-9~-][a-z0-9._~-]*$/;
|
|
164
|
+
const RELATIVE_IMPORT = /(?:\bfrom\s*|\bimport\s*\(?\s*|\brequire\s*\(\s*)["'](\.\.?\/[^"']*)["']/g;
|
|
165
|
+
const BUILTINS = new Set(builtinModules);
|
|
166
|
+
/** Never fetched: the shim replaces them. */
|
|
167
|
+
const SHIMMED = new Set(["@graphprotocol/graph-ts", "matchstick-as"]);
|
|
168
|
+
const MAX_SCANNED_FILES = 2_000;
|
|
169
|
+
|
|
170
|
+
function packageName(specifier: string): string | null {
|
|
171
|
+
if (specifier.startsWith("node:") || specifier.startsWith("#")) return null;
|
|
172
|
+
const parts = specifier.split("/");
|
|
173
|
+
const name = specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0];
|
|
174
|
+
if (!name || !NPM_NAME.test(name) || BUILTINS.has(name) || SHIMMED.has(name)) return null;
|
|
175
|
+
return name;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Whether Node would find `name` from `fromDir`: a node_modules there or in any parent. */
|
|
179
|
+
function resolvableFrom(fromDir: string, name: string): boolean {
|
|
180
|
+
for (let dir = path.resolve(fromDir); ; dir = path.dirname(dir)) {
|
|
181
|
+
if (existsSync(path.join(dir, "node_modules", name, "package.json"))) return true;
|
|
182
|
+
if (path.dirname(dir) === dir) return false;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* The npm packages the mappings import that the project can't resolve, found by
|
|
188
|
+
* walking the mapping files' relative imports. `generated/` is not walked: it
|
|
189
|
+
* imports only graph-ts.
|
|
190
|
+
*/
|
|
191
|
+
export function missingMappingPackages(root: string, mappingFiles: string[]): string[] {
|
|
192
|
+
const queue = [...new Set(mappingFiles)].map((file) => path.resolve(root, file));
|
|
193
|
+
const seen = new Set(queue);
|
|
194
|
+
const missing = new Set<string>();
|
|
195
|
+
for (let index = 0; index < queue.length && index < MAX_SCANNED_FILES; index++) {
|
|
196
|
+
const file = queue[index];
|
|
197
|
+
let source: string;
|
|
198
|
+
try {
|
|
199
|
+
source = readFileSync(file, "utf8");
|
|
200
|
+
} catch {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
for (const [, specifier] of source.matchAll(BARE_IMPORT)) {
|
|
204
|
+
const name = packageName(specifier);
|
|
205
|
+
if (name && !resolvableFrom(path.dirname(file), name)) missing.add(name);
|
|
206
|
+
}
|
|
207
|
+
for (const [, specifier] of source.matchAll(RELATIVE_IMPORT)) {
|
|
208
|
+
const target = path.resolve(path.dirname(file), specifier);
|
|
209
|
+
for (const candidate of [target, `${target}.ts`, path.join(target, "index.ts")]) {
|
|
210
|
+
if (!seen.has(candidate) && existsSync(candidate) && statSync(candidate).isFile()) {
|
|
211
|
+
seen.add(candidate);
|
|
212
|
+
queue.push(candidate);
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return [...missing].sort();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Fetches `names` into the cache, each at the version the project declares,
|
|
223
|
+
* falling back to the latest when that version is gone (ETARGET). Returns the
|
|
224
|
+
* `node_modules` directory holding them, or null when nothing installed. A
|
|
225
|
+
* package that can't be fetched at all (a workspace package that isn't on the
|
|
226
|
+
* registry) is left out, so the mapping that imports it reports it by name.
|
|
227
|
+
*/
|
|
228
|
+
export async function fetchMappingPackages(root: string, names: string[]): Promise<string | null> {
|
|
229
|
+
if (names.length === 0) return null;
|
|
230
|
+
const specs = names.map((name) => {
|
|
231
|
+
const version = declaredVersion(root, name);
|
|
232
|
+
return version ? `${name}@${version}` : name;
|
|
233
|
+
});
|
|
234
|
+
const key = createHash("sha1").update(specs.join("\n")).digest("hex").slice(0, 12);
|
|
235
|
+
const dir = path.join(cacheDir(), "packages", key);
|
|
236
|
+
const modules = path.join(dir, "node_modules");
|
|
237
|
+
if (existsSync(path.join(dir, ".envio-installed"))) return modules;
|
|
238
|
+
|
|
239
|
+
info(`Envio Subgraph is fetching ${specs.join(", ")} into ${dir}: the project's own node_modules lacks ${names.length === 1 ? "it" : "them"}.`);
|
|
240
|
+
try {
|
|
241
|
+
await npmInstallInto(dir, specs);
|
|
242
|
+
return modules;
|
|
243
|
+
} catch {
|
|
244
|
+
// One bad spec fails the whole install; find which.
|
|
245
|
+
}
|
|
246
|
+
const installed: string[] = [];
|
|
247
|
+
for (const [index, spec] of specs.entries()) {
|
|
248
|
+
const one = path.join(cacheDir(), "packages", createHash("sha1").update(spec).digest("hex").slice(0, 12));
|
|
249
|
+
for (const attempt of spec === names[index] ? [spec] : [spec, names[index]]) {
|
|
250
|
+
try {
|
|
251
|
+
await npmInstallInto(one, [attempt]);
|
|
252
|
+
installed.push(one);
|
|
253
|
+
break;
|
|
254
|
+
} catch (error) {
|
|
255
|
+
if (attempt === names[index]) {
|
|
256
|
+
warn(`Envio Subgraph could not fetch ${names[index]} (${String(error).split("\n")[0]}); a mapping importing it will fail by name.`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (installed.length === 0) return null;
|
|
262
|
+
// Merge the per-package installs into one node_modules, so resolution has a single place to look.
|
|
263
|
+
const merged = path.join(dir, "node_modules");
|
|
264
|
+
mkdirSync(merged, { recursive: true });
|
|
265
|
+
for (const one of installed) {
|
|
266
|
+
for (const entry of readdirSync(path.join(one, "node_modules"))) {
|
|
267
|
+
cpSync(path.join(one, "node_modules", entry), path.join(merged, entry), { recursive: true, force: false });
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
writeFileSync(path.join(dir, ".envio-installed"), specs.join("\n"));
|
|
271
|
+
return modules;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// --- graph-cli ----------------------------------------------------------------
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* The newest graph-cli whose own dependencies still install, and the one the
|
|
278
|
+
* generated code of older subgraphs is closest to: 0.51 has none of the
|
|
279
|
+
* dead git pins that 0.45 and older do (hugomrdias/concat-stream, the
|
|
280
|
+
* edgeandnode/gluegun fork) and still accepts a schema graph-cli 0.9x rejects
|
|
281
|
+
* (`@entity` without `immutable`, nullable list members are the next gate).
|
|
282
|
+
*/
|
|
283
|
+
export const OLDEST_INSTALLABLE_GRAPH_CLI = "0.51.0";
|
|
284
|
+
/** For manifests newer than 0.51 reads (specVersion 1.x, indexerHints). */
|
|
285
|
+
export const NEWEST_GRAPH_CLI = "0.98.1";
|
|
286
|
+
|
|
287
|
+
/** The first x.y.z in a spec, as numbers. */
|
|
288
|
+
function leadingVersion(spec: string): [number, number, number] | null {
|
|
289
|
+
const match = spec.match(/(\d+)\.(\d+)\.(\d+)/);
|
|
290
|
+
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function atLeast(version: [number, number, number], minimum: [number, number, number]): boolean {
|
|
294
|
+
for (let index = 0; index < 3; index++) {
|
|
295
|
+
if (version[index] !== minimum[index]) return version[index] > minimum[index];
|
|
296
|
+
}
|
|
297
|
+
return true;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* graph-cli versions to try, best first: the project's own when it is new
|
|
302
|
+
* enough to install (a version the repo's author ran), then the two pins.
|
|
303
|
+
*/
|
|
304
|
+
export function graphCliVersions(root: string): string[] {
|
|
305
|
+
const versions: string[] = [];
|
|
306
|
+
const declared = declaredVersion(root, "@graphprotocol/graph-cli");
|
|
307
|
+
const parsed = declared ? leadingVersion(declared) : null;
|
|
308
|
+
if (declared && parsed && atLeast(parsed, [0, 50, 0])) versions.push(declared);
|
|
309
|
+
versions.push(OLDEST_INSTALLABLE_GRAPH_CLI, NEWEST_GRAPH_CLI);
|
|
310
|
+
return [...new Set(versions)];
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** The package directory of `@graphprotocol/graph-cli@version` in the cache, fetching it first if need be. */
|
|
314
|
+
export async function fetchGraphCli(version: string): Promise<string | null> {
|
|
315
|
+
const dir = path.join(cacheDir(), "graph-cli", version.replace(/[^0-9A-Za-z.^~<>=-]/g, "_"));
|
|
316
|
+
const packageDir = path.join(dir, "node_modules", "@graphprotocol", "graph-cli");
|
|
317
|
+
if (!existsSync(path.join(dir, ".envio-installed"))) {
|
|
318
|
+
info(`Envio Subgraph is fetching @graphprotocol/graph-cli@${version} into ${dir} to generate code: the project's own isn't usable.`);
|
|
319
|
+
try {
|
|
320
|
+
await npmInstallInto(dir, [`@graphprotocol/graph-cli@${version}`]);
|
|
321
|
+
} catch (error) {
|
|
322
|
+
warn(`Envio Subgraph could not fetch @graphprotocol/graph-cli@${version}: ${String(error).split("\n").slice(0, 3).join(" ")}`);
|
|
323
|
+
return null;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return existsSync(path.join(packageDir, "package.json")) ? packageDir : null;
|
|
327
|
+
}
|