@openzeppelin/upgrades-core 1.9.1

package/src/link-refs.ts

@@ -0,0 +1,41 @@
+import { SolcBytecode } from './solc-api';
+
+export interface LinkReference {
+  src: string;
+  name: string;
+  length: number;
+  start: number;
+  placeholder: string;
+}
+
+export function extractLinkReferences(bytecode: SolcBytecode): LinkReference[] {
+  const linkRefs: LinkReference[] = [];
+  const { linkReferences } = bytecode;
+  for (const source of Object.keys(linkReferences)) {
+    for (const name of Object.keys(linkReferences[source])) {
+      for (const { length, start } of linkReferences[source][name]) {
+        const placeholder = bytecode.object.substr(start * 2, length * 2);
+        linkRefs.push({
+          src: source,
+          name,
+          length,
+          start,
+          placeholder,
+        });
+      }
+    }
+  }
+  return linkRefs;
+}
+
+export function unlinkBytecode(bytecode: string, linkReferences: LinkReference[]): string {
+  let unlinkedBytecode: string = bytecode.replace(/^0x/, '');
+  for (const linkRef of linkReferences) {
+    const { length, start, placeholder } = linkRef;
+    if ((start + length) * 2 <= unlinkedBytecode.length) {
+      unlinkedBytecode =
+        unlinkedBytecode.substr(0, start * 2) + placeholder + unlinkedBytecode.substr((start + length) * 2);
+    }
+  }
+  return '0x' + unlinkedBytecode;
+}

package/src/manifest-storage-layout.test.ts

@@ -0,0 +1,118 @@
+import _test, { TestInterface } from 'ava';
+import assert from 'assert';
+
+import { artifacts } from 'hardhat';
+
+import { solcInputOutputDecoder } from './src-decoder';
+import { validate, ValidationRunData } from './validate/run';
+import { getStorageLayout } from './validate/query';
+import { normalizeValidationData, ValidationData } from './validate/data';
+import { StorageLayout } from './storage/layout';
+
+import { Manifest, ManifestData } from './manifest';
+import { getUpdatedStorageLayout, getStorageLayoutForAddress } from './manifest-storage-layout';
+
+interface Context {
+  validationRun: ValidationRunData;
+  validationData: ValidationData;
+}
+
+const test = _test as TestInterface<Context>;
+
+test.before(async t => {
+  const buildInfo = await artifacts.getBuildInfo('contracts/test/ManifestMigrate.sol:ManifestMigrateUnique');
+  assert(buildInfo !== undefined, 'Build info not found');
+  const decodeSrc = solcInputOutputDecoder(buildInfo.input, buildInfo.output);
+  t.context.validationRun = validate(buildInfo.output, decodeSrc);
+  t.context.validationData = normalizeValidationData([t.context.validationRun]);
+});
+
+test('getStorageLayoutForAddress - update layout', async t => {
+  const { version } = t.context.validationRun['ManifestMigrateUnique'];
+  assert(version !== undefined);
+  const updatedLayout = getStorageLayout(t.context.validationData, version);
+  const outdatedLayout = removeStorageLayoutMembers(updatedLayout);
+  const address = '0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9';
+  const manifest = mockManifest({
+    manifestVersion: '3.2',
+    impls: {
+      [version.withoutMetadata]: {
+        address,
+        txHash: '0x6580b51f3edcacacf30d7b4140e4022b65d2a5ba7cbe7e4d91397f4c3b5e8a6b',
+        layout: outdatedLayout,
+      },
+    },
+    proxies: [],
+  });
+  const layout = await getStorageLayoutForAddress(manifest, t.context.validationData, address);
+  t.deepEqual(layout, updatedLayout);
+  t.like(manifest.data, {
+    impls: {
+      [version.withoutMetadata]: {
+        address,
+        txHash: '0x6580b51f3edcacacf30d7b4140e4022b65d2a5ba7cbe7e4d91397f4c3b5e8a6b',
+        layout: updatedLayout,
+      },
+    },
+    proxies: [],
+  });
+});
+
+test('getUpdatedLayout - unique layout match', async t => {
+  const { version } = t.context.validationRun['ManifestMigrateUnique'];
+  assert(version !== undefined);
+  const targetLayout = getStorageLayout(t.context.validationData, version);
+  const outdatedLayout = removeStorageLayoutMembers(targetLayout);
+  const updatedLayout = getUpdatedStorageLayout(t.context.validationData, version.withoutMetadata, outdatedLayout);
+  t.deepEqual(updatedLayout, targetLayout);
+});
+
+test('getUpdatedLayout - multiple unambiguous layout matches', async t => {
+  const { version: version1 } = t.context.validationRun['ManifestMigrateUnambiguous1'];
+  const { version: version2 } = t.context.validationRun['ManifestMigrateUnambiguous2'];
+  assert(version1 !== undefined && version2 !== undefined);
+  t.is(version1.withoutMetadata, version2.withoutMetadata, 'version is meant to be ambiguous');
+  t.not(version1.withMetadata, version2.withMetadata, 'version with metadata should be different');
+  const targetLayout = getStorageLayout(t.context.validationData, version1);
+  const outdatedLayout = removeStorageLayoutMembers(targetLayout);
+  const updatedLayout = getUpdatedStorageLayout(t.context.validationData, version1.withoutMetadata, outdatedLayout);
+  t.deepEqual(updatedLayout, targetLayout);
+});
+
+test('getUpdatedLayout - multiple ambiguous layout matches', async t => {
+  const { version: version1 } = t.context.validationRun['ManifestMigrateAmbiguous1'];
+  const { version: version2 } = t.context.validationRun['ManifestMigrateAmbiguous2'];
+  assert(version1 !== undefined && version2 !== undefined);
+  t.is(version1.withoutMetadata, version2.withoutMetadata, 'version is meant to be ambiguous');
+  t.not(version1.withMetadata, version2.withMetadata, 'version with metadata should be different');
+  const targetLayout = getStorageLayout(t.context.validationData, version1);
+  const outdatedLayout = removeStorageLayoutMembers(targetLayout);
+  const updatedLayout = getUpdatedStorageLayout(t.context.validationData, version1.withoutMetadata, outdatedLayout);
+  t.is(updatedLayout, undefined);
+});
+
+function mockManifest(data: ManifestData) {
+  type Mocked = 'read' | 'write' | 'lockedRun';
+  const manifest: Pick<Manifest, Mocked> & { data: ManifestData } = {
+    data,
+    async read() {
+      return this.data;
+    },
+    async write(data: ManifestData) {
+      this.data = data;
+    },
+    async lockedRun<T>(cb: () => Promise<T>) {
+      return cb();
+    },
+  };
+  return manifest as Manifest & { data: ManifestData };
+}
+
+// Simulate a layout from a version without struct/enum members
+function removeStorageLayoutMembers(layout: StorageLayout): StorageLayout {
+  const res = { ...layout, types: { ...layout.types } };
+  for (const id in res.types) {
+    res.types[id] = { ...layout.types[id], members: undefined };
+  }
+  return res;
+}

package/src/manifest-storage-layout.ts

@@ -0,0 +1,107 @@
+import { Manifest } from './manifest';
+import { isCurrentLayoutVersion } from './storage/extract';
+import { StorageLayout, isStructMembers } from './storage/layout';
+import { ValidationData } from './validate/data';
+import { findVersionWithoutMetadataMatches, unfoldStorageLayout } from './validate/query';
+import { stabilizeTypeIdentifier } from './utils/type-id';
+import { DeepArray, deepEqual } from './utils/deep-array';
+
+export async function getStorageLayoutForAddress(
+  manifest: Manifest,
+  validations: ValidationData,
+  implAddress: string,
+): Promise<StorageLayout> {
+  const data = await manifest.read();
+  const versionWithoutMetadata = Object.keys(data.impls).find(v => data.impls[v]?.address === implAddress);
+  if (versionWithoutMetadata === undefined) {
+    throw new Error(`Deployment at address ${implAddress} is not registered`);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+  const { layout } = data.impls[versionWithoutMetadata]!;
+
+  if (isCurrentLayoutVersion(layout)) {
+    return layout;
+  } else {
+    const updatedLayout = getUpdatedStorageLayout(validations, versionWithoutMetadata, layout);
+
+    if (updatedLayout === undefined) {
+      return layout;
+    } else {
+      await manifest.lockedRun(async () => {
+        const data = await manifest.read();
+        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+        data.impls[versionWithoutMetadata]!.layout = updatedLayout;
+        await manifest.write(data);
+      });
+      return updatedLayout;
+    }
+  }
+}
+
+// This function is used to retrieve the updated storage layout for an impl
+// contract from the Manifest file, for which we only have the version hash
+// without metadata and a storage layout. The storage layout is updated in the
+// sense that it is updated to include newer information that hadn't been
+// extracted when the impl contract was deployed, such as struct members.
+export function getUpdatedStorageLayout(
+  data: ValidationData,
+  versionWithoutMetadata: string,
+  layout: StorageLayout,
+): StorageLayout | undefined {
+  // In this map we store the candidate storage layouts, based on having the
+  // same versionWithoutMetadata and a sufficiently similar layout. The keys of
+  // the map are the versionWithMetadata of each contract, to avoid storing the
+  // same contract multiple times.
+  const sameLayout = new Map<string, StorageLayout>();
+
+  outer: for (const [contractName, validationRun] of findVersionWithoutMetadataMatches(data, versionWithoutMetadata)) {
+    const updatedLayout = unfoldStorageLayout(validationRun, contractName);
+
+    // Check that the layout roughly matches the one we already have.
+    if (updatedLayout.storage.length !== layout.storage.length) {
+      continue;
+    }
+    for (const [i, item] of layout.storage.entries()) {
+      const updatedItem = updatedLayout.storage[i];
+      if (item.label !== updatedItem.label) {
+        continue outer;
+      }
+      if (stabilizeTypeIdentifier(item.type) !== stabilizeTypeIdentifier(updatedItem.type)) {
+        continue outer;
+      }
+    }
+
+    const { version } = validationRun[contractName];
+    if (version && !sameLayout.has(version.withMetadata)) {
+      sameLayout.set(version.withMetadata, updatedLayout);
+    }
+  }
+
+  if (sameLayout.size > 1) {
+    // Reject multiple matches unless they all have exactly the same layout.
+    const typeMembers = new Map<string, DeepArray<string>>();
+    for (const { types } of sameLayout.values()) {
+      for (const [typeId, item] of Object.entries(types)) {
+        if (item.members === undefined) {
+          continue;
+        }
+
+        const members: DeepArray<string> = isStructMembers(item.members)
+          ? item.members.map(m => [stabilizeTypeIdentifier(m.type), m.label])
+          : item.members;
+        const stableId = stabilizeTypeIdentifier(typeId);
+
+        if (!typeMembers.has(stableId)) {
+          typeMembers.set(stableId, members);
+        } else if (!deepEqual(members, typeMembers.get(stableId))) {
+          return undefined;
+        }
+      }
+    }
+  }
+
+  if (sameLayout.size > 0) {
+    const [updatedLayout] = sameLayout.values();
+    return updatedLayout;
+  }
+}

package/src/manifest.test.ts

@@ -0,0 +1,13 @@
+import test from 'ava';
+import { Manifest } from './manifest';
+
+test('manifest name for a known network', t => {
+  const manifest = new Manifest(1);
+  t.is(manifest.file, '.openzeppelin/mainnet.json');
+});
+
+test('manifest name for an unknown network', t => {
+  const id = 55555;
+  const manifest = new Manifest(id);
+  t.is(manifest.file, `.openzeppelin/unknown-${id}.json`);
+});

package/src/manifest.ts

@@ -0,0 +1,178 @@
+import path from 'path';
+import { promises as fs } from 'fs';
+import { EthereumProvider, getChainId, networkNames } from './provider';
+import lockfile from 'proper-lockfile';
+import { compare as compareVersions } from 'compare-versions';
+
+import type { Deployment } from './deployment';
+import type { StorageLayout } from './storage';
+import { pick } from './utils/pick';
+import { mapValues } from './utils/map-values';
+
+const currentManifestVersion = '3.2';
+
+export interface ManifestData {
+  manifestVersion: string;
+  impls: {
+    [version in string]?: ImplDeployment;
+  };
+  proxies: ProxyDeployment[];
+  admin?: Deployment;
+}
+
+export interface ImplDeployment extends Deployment {
+  layout: StorageLayout;
+}
+
+export interface ProxyDeployment extends Deployment {
+  kind: 'uups' | 'transparent';
+}
+
+function defaultManifest(): ManifestData {
+  return {
+    manifestVersion: currentManifestVersion,
+    impls: {},
+    proxies: [],
+  };
+}
+
+const manifestDir = '.openzeppelin';
+
+export class Manifest {
+  readonly file: string;
+  private locked = false;
+
+  static async forNetwork(provider: EthereumProvider): Promise<Manifest> {
+    return new Manifest(await getChainId(provider));
+  }
+
+  constructor(readonly chainId: number) {
+    const name = networkNames[chainId] ?? `unknown-${chainId}`;
+    this.file = path.join(manifestDir, `${name}.json`);
+  }
+
+  async getAdmin(): Promise<Deployment | undefined> {
+    return (await this.read()).admin;
+  }
+
+  async getDeploymentFromAddress(address: string): Promise<ImplDeployment> {
+    const data = await this.read();
+    const deployment = Object.values(data.impls).find(d => d?.address === address);
+    if (deployment === undefined) {
+      throw new DeploymentNotFound(`Deployment at address ${address} is not registered`);
+    }
+    return deployment;
+  }
+
+  async getProxyFromAddress(address: string): Promise<ProxyDeployment> {
+    const data = await this.read();
+    const deployment = data.proxies.find(d => d?.address === address);
+    if (deployment === undefined) {
+      throw new DeploymentNotFound(`Proxy at address ${address} is not registered`);
+    }
+    return deployment;
+  }
+
+  async addProxy(proxy: ProxyDeployment): Promise<void> {
+    await this.lockedRun(async () => {
+      const data = await this.read();
+      const existing = data.proxies.findIndex(p => p.address === proxy.address);
+      if (existing >= 0) {
+        data.proxies.splice(existing, 1);
+      }
+      data.proxies.push(proxy);
+      await this.write(data);
+    });
+  }
+
+  async read(): Promise<ManifestData> {
+    const release = this.locked ? undefined : await this.lock();
+    try {
+      const data = JSON.parse(await fs.readFile(this.file, 'utf8')) as ManifestData;
+      return validateOrUpdateManifestVersion(data);
+    } catch (e) {
+      if (e.code === 'ENOENT') {
+        return defaultManifest();
+      } else {
+        throw e;
+      }
+    } finally {
+      await release?.();
+    }
+  }
+
+  async write(data: ManifestData): Promise<void> {
+    if (!this.locked) {
+      throw new Error('Manifest must be locked');
+    }
+    const normalized = normalizeManifestData(data);
+    await fs.writeFile(this.file, JSON.stringify(normalized, null, 2) + '\n');
+  }
+
+  async lockedRun<T>(cb: () => Promise<T>): Promise<T> {
+    if (this.locked) {
+      throw new Error('Manifest is already locked');
+    }
+    const release = await this.lock();
+    try {
+      return await cb();
+    } finally {
+      await release();
+    }
+  }
+
+  private async lock() {
+    await fs.mkdir(path.dirname(this.file), { recursive: true });
+    const release = await lockfile.lock(this.file, { retries: 3, realpath: false });
+    this.locked = true;
+    return async () => {
+      await release();
+      this.locked = false;
+    };
+  }
+}
+
+function validateOrUpdateManifestVersion(data: ManifestData): ManifestData {
+  if (typeof data.manifestVersion !== 'string') {
+    throw new Error('Manifest version is missing');
+  } else if (compareVersions(data.manifestVersion, '3.0', '<')) {
+    throw new Error('Found a manifest file for OpenZeppelin CLI. An automated migration is not yet available.');
+  } else if (compareVersions(data.manifestVersion, currentManifestVersion, '<')) {
+    return migrateManifest(data);
+  } else if (data.manifestVersion === currentManifestVersion) {
+    return data;
+  } else {
+    throw new Error(`Unknown value for manifest version (${data.manifestVersion})`);
+  }
+}
+
+export function migrateManifest(data: ManifestData): ManifestData {
+  switch (data.manifestVersion) {
+    case '3.0':
+    case '3.1':
+      data.manifestVersion = currentManifestVersion;
+      data.proxies = [];
+      return data;
+    default:
+      throw new Error('Manifest migration not available');
+  }
+}
+
+export class DeploymentNotFound extends Error {}
+
+function normalizeManifestData(input: ManifestData): ManifestData {
+  return {
+    ...pick(input, ['manifestVersion', 'admin']),
+    proxies: input.proxies.map(p => normalizeDeployment(p, ['kind'])),
+    impls: mapValues(input.impls, i => i && normalizeDeployment(i, ['layout'])),
+  };
+}
+
+function normalizeDeployment<D extends Deployment>(input: D): Deployment;
+function normalizeDeployment<D extends Deployment, K extends keyof D>(input: D, include: K[]): Deployment & Pick<D, K>;
+function normalizeDeployment<D extends Deployment, K extends keyof D>(
+  input: D,
+  include: K[] = [],
+): Deployment & Pick<D, K> {
+  return pick(input, ['address', 'txHash', ...include]);
+}