@hyperweb/telescope 1.15.0

package/src/protod/recursive.spec.ts

@@ -0,0 +1,164 @@
+import { TelescopeInput } from "..";
+import { TelescopeBuilder } from "../builder";
+import { TelescopeOptions } from "@cosmology/types";
+
+import { clone, cloneAll, extractProto } from "./recursive";
+import { removeFolder } from "./utils";
+
+import { join } from "path";
+
+const GIT_MOUDLES = "git-modules";
+let gitModules = join(
+  __dirname,
+  "../../../../__fixtures__/misc/proto-download",
+  GIT_MOUDLES
+);
+
+const GIT_MOUDLES_ALL = "git-modules-all";
+let gitModulesAll = join(
+  __dirname,
+  "../../../../__fixtures__/misc/proto-download",
+  GIT_MOUDLES_ALL
+);
+
+const PROTO_FOLDER = "proto";
+let protoFolder = join(
+  __dirname,
+  "../../../../__fixtures__/misc/proto-download",
+  PROTO_FOLDER
+);
+const CODEGEN_FOLDER = "codegen";
+let codegenFolder = join(
+  __dirname,
+  "../../../../__fixtures__/misc/proto-download",
+  CODEGEN_FOLDER
+);
+
+const options: TelescopeOptions = {
+  classesUseArrowFunctions: true,
+  env: "v-next",
+  useSDKTypes: false,
+  prototypes: {
+    useOptionalNullable: true,
+    fieldDefaultIsOptional: false,
+    enableRegistryLoader: false,
+    enableMessageComposer: false,
+    enabled: true,
+    parser: {
+      keepCase: false,
+    },
+    methods: {
+      fromJSON: false,
+      toJSON: false,
+      encode: true,
+      decode: true,
+      fromPartial: true,
+      toAmino: false,
+      fromAmino: false,
+      fromProto: false,
+      toProto: false,
+    },
+    addTypeUrlToDecoders: false,
+    addTypeUrlToObjects: true,
+    addAminoTypeToObjects: true,
+    typingsFormat: {
+      duration: "duration",
+      timestamp: "date",
+      useExact: false,
+      useDeepPartial: true,
+      num64: "bigint",
+      customTypes: {
+        useCosmosSDKDec: false,
+      },
+      useTelescopeGeneratedType: true,
+    },
+  },
+  bundle: {
+    enabled: false,
+  },
+  stargateClients: {
+    enabled: false,
+  },
+  lcdClients: {
+    enabled: false,
+  },
+  rpcClients: {
+    enabled: false,
+    inline: false,
+    extensions: false,
+    camelCase: true,
+    enabledServices: ["Query"],
+  },
+  interfaces: {
+    enabled: false,
+  },
+  aminoEncoding: {
+    enabled: false,
+  },
+};
+
+const input: TelescopeInput = {
+  outPath: codegenFolder,
+  protoDirs: [protoFolder],
+  options,
+};
+
+const testSwitch = false;
+
+describe("Test `cloneAll`", () => {
+  (testSwitch ? it : it.skip)("should download `injective`, `ibc-go` to `./git-modules`", async () => {
+    removeFolder(gitModulesAll);
+    const result = await cloneAll({
+      repos: [
+        { owner: "cosmos", repo: "ibc-go" },
+        { owner: "injectivelabs", repo: "sdk-go" },
+      ],
+      gitModulesDir: gitModulesAll,
+      protoDirMapping: {
+        "gogo/protobuf/master": ".",
+        "googleapis/googleapis/master": ".",
+        "protocolbuffers/protobuf/main": "src",
+      },
+      ssh: true,
+    });
+
+    console.log(result);
+  });
+});
+
+describe("Test `clone`", () => {
+  (testSwitch ? it : it.skip)("should download `cosmos-sdk`, `googleapis`, `protobuf` to `./git-modules` and extract `cosmos/bank/v1beta1/tx.proto` to `./proto`", async () => {
+    removeFolder(gitModules);
+    const result = await clone({
+      owner: "cosmos",
+      repo: "cosmos-sdk",
+      gitModulesDir: gitModules,
+      branch: "release/v0.50.x",
+      protoDirMapping: {
+        "gogo/protobuf/master": ".",
+        "googleapis/googleapis/master": ".",
+        "protocolbuffers/protobuf/main": "src",
+      },
+      ssh: true,
+    });
+
+    if (result) {
+      removeFolder(protoFolder);
+      extractProto({
+        targets: [
+          "cosmos/bank/v1beta1/tx.proto",
+          "cosmos/gov/**/*.proto",
+          "cosmos/authz/**/*.proto",
+        ],
+        sources: result,
+        outDir: protoFolder,
+      });
+    }
+  });
+
+  (testSwitch ? it : it.skip)("should successfully generate code in `./codegen`", async () => {
+    removeFolder(codegenFolder);
+    const telescope = new TelescopeBuilder(input);
+    await telescope.build();
+  });
+});

package/src/protod/recursive.ts

@@ -0,0 +1,180 @@
+import { getAllBufDeps } from "./bufbuild";
+import { GitRepo } from "./git-repo";
+import {
+  CloneOptions,
+  GitInfo,
+  ProtoCopyOptions,
+  CloneAllOptions,
+} from "./types";
+import { join, dirname, resolve, relative } from "path";
+import { getCorrespondingGit, makeDir, parseProtoFile } from "./utils";
+import fs from "fs";
+import { crossGlob as glob } from '@cosmology/utils';
+
+export async function cloneAll({
+  repos,
+  gitModulesDir,
+  protoDirMapping,
+  ssh,
+}: CloneAllOptions) {
+  let clonedResult: Record<string, GitInfo> = {};
+
+  for (const { owner, repo, branch } of repos) {
+    const cloned = await clone({
+      owner,
+      repo,
+      branch,
+      gitModulesDir,
+      protoDirMapping,
+      ssh,
+      cloned: clonedResult,
+    });
+
+    for (const [key, value] of Object.entries(cloned)) {
+      clonedResult[key] = value;
+    }
+  }
+
+  return clonedResult;
+}
+
+export async function clone({
+  owner,
+  repo,
+  branch,
+  gitModulesDir: outDir,
+  protoDirMapping,
+  ssh,
+  cloned,
+}: CloneOptions) {
+  let clonedResult: Record<string, GitInfo> = cloned ?? {};
+
+  const gitRepo = new GitRepo(owner, repo, ssh);
+  const gitBranch = branch ?? (await gitRepo.getMainBranchName());
+  const resultKey = `${owner}/${repo}/${gitBranch}`;
+
+  if (clonedResult[resultKey]) {
+    console.log(`Skip cloning ${resultKey}`);
+    return clonedResult;
+  }
+
+  const gitDir = gitRepo.clone(gitBranch, 1, outDir);
+  const protoDir =
+    protoDirMapping?.[`${owner}/${repo}/${gitBranch}`] ?? "proto";
+  clonedResult[`${owner}/${repo}/${gitBranch}`] = {
+    owner,
+    repo,
+    branch: gitBranch,
+    protoDir,
+    protoPath: resolve(`${outDir}/${owner}/${repo}/${gitBranch}/${protoDir}`),
+  };
+  const bufDeps = await getAllBufDeps(gitDir);
+  await Promise.all(
+    bufDeps.map(async (bufRepo) => {
+      const gitRepos = getCorrespondingGit(bufRepo);
+      await Promise.all(
+        gitRepos.map(async (gitRepo) => {
+          const gitRepoObj = new GitRepo(gitRepo.owner, gitRepo.repo, ssh);
+          const branch = await gitRepoObj.getMainBranchName();
+          const depsClonedResult = await clone({
+            ...gitRepo,
+            gitModulesDir: outDir,
+            branch,
+            protoDirMapping,
+            ssh,
+            cloned: clonedResult,
+          });
+          for (const [key, value] of Object.entries(depsClonedResult)) {
+            clonedResult[key] = value;
+          }
+        })
+      );
+    })
+  );
+
+  return clonedResult;
+}
+
+export function extractProto({ sources, targets, outDir }: ProtoCopyOptions) {
+  const extractProtoFiles: { sourceFile: string; target: string }[] =
+    extractProtoFromDirs({
+      targets,
+      sources,
+    });
+
+  extractProtoFiles.map(({ sourceFile, target }) => {
+    const targetFile = join(outDir, target);
+    const deepTargetDir = dirname(targetFile);
+    makeDir(deepTargetDir);
+    fs.copyFileSync(sourceFile, targetFile);
+    console.info(`Copied ${target} from ${sourceFile.replace(target, "")}`);
+  });
+}
+
+function extractProtoFromDirs({
+  targets,
+  sources,
+}: {
+  targets: string[];
+  sources: Record<string, GitInfo>;
+}) {
+  const extractProtoFiles: { sourceFile: string; target: string }[] = [];
+  const existingFiles = new Set();
+
+  if (!targets || targets.length === 0) {
+    return [];
+  }
+
+  if (!sources || Object.keys(sources).length === 0) {
+    return [];
+  }
+
+  for (const target of targets) {
+    for (const source of Object.values(sources)) {
+      const sourceProtoPath = resolve(source.protoPath);
+      const files = glob(join(sourceProtoPath, target));
+
+      extractProtoFiles.push(
+        ...files
+          .map((file) => {
+            const copyTarget = relative(sourceProtoPath, file);
+            const duplicate = existingFiles.has(copyTarget);
+            existingFiles.add(copyTarget);
+            if (!duplicate) {
+              const resultFiles = [
+                {
+                  sourceFile: file,
+                  target: copyTarget,
+                },
+              ];
+
+              const newTargets = parseProtoFile(file);
+
+              if (newTargets && newTargets.length > 0) {
+                const deps = extractProtoFromDirs({
+                  targets: newTargets,
+                  sources,
+                });
+
+                const filteredDeps = deps?.filter((dep) => {
+                  const depDuplicate = existingFiles.has(dep.target);
+                  existingFiles.add(dep.target);
+                  return !depDuplicate;
+                });
+
+                if (filteredDeps && filteredDeps.length > 0) {
+                  resultFiles.push(...filteredDeps);
+                }
+              }
+
+              return resultFiles;
+            }
+          })
+          .flat()
+          .filter(Boolean)
+      );
+    }
+  }
+
+  return extractProtoFiles;
+}

package/src/protod/test-data/.protod.config.json

@@ -0,0 +1,42 @@
+{
+      "repos": [
+        { "owner": "cosmos", "repo": "cosmos-sdk", "branch": "release/v0.50.x" },
+        { "owner": "cosmos", "repo": "ibc-go" },
+        { "owner": "injectivelabs", "repo": "sdk-go" }
+      ],
+      "protoDirMapping": {
+        "gogo/protobuf/master": ".",
+        "googleapis/googleapis/master": ".",
+        "protocolbuffers/protobuf/main": "src"
+      },
+      "outDir": "../../__fixtures__/misc/proto-download/testproto",
+      "ssh": true,
+      "tempRepoDir": "git-modules",
+      "targets": [
+        "cosmos/auth/v1beta1/auth.proto",
+        "cosmos/vesting/v1beta1/vesting.proto",
+        "cosmos/crypto/secp256k1/keys.proto",
+        "cosmos/crypto/secp256r1/keys.proto",
+        "cosmos/crypto/ed25519/keys.proto",
+        "cosmos/authz/v1beta1/tx.proto",
+        "cosmos/bank/v1beta1/tx.proto",
+        "cosmos/distribution/v1beta1/tx.proto",
+        "cosmos/feegrant/v1beta1/tx.proto",
+        "cosmos/gov/v1beta1/tx.proto",
+        "cosmos/gov/v1/tx.proto",
+        "cosmos/group/v1/tx.proto",
+        "cosmos/staking/v1beta1/tx.proto",
+        "cosmos/vesting/v1beta1/tx.proto",
+        "cosmos/auth/v1beta1/query.proto",
+        "cosmos/bank/v1beta1/query.proto",
+        "cosmos/staking/v1beta1/query.proto",
+        "cosmos/gov/v1beta1/query.proto",
+        "cosmwasm/wasm/v1/tx.proto",
+        "cosmwasm/wasm/v1/query.proto",
+        "ibc/applications/transfer/v1/tx.proto",
+        "ibc/core/channel/v1/tx.proto",
+        "ibc/core/client/v1/tx.proto",
+        "ibc/core/connection/v1/tx.proto",
+        "injective/types/v1beta1/account.proto"
+      ]
+}

package/src/protod/test-data/buf.lock

@@ -0,0 +1,23 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-proto
+    commit: 1935555c206d4afb9e94615dfd0fad31
+    digest: shake256:c74d91a3ac7ae07d579e90eee33abf9b29664047ac8816500cf22c081fec0d72d62c89ce0bebafc1f6fec7aa5315be72606717740ca95007248425102c365377
+  - remote: buf.build
+    owner: cosmos
+    repository: gogo-proto
+    commit: 5e5b9fdd01804356895f8f79a6f1ddc1
+    digest: shake256:0b85da49e2e5f9ebc4806eae058e2f56096ff3b1c59d1fb7c190413dd15f45dd456f0b69ced9059341c80795d2b6c943de15b120a9e0308b499e43e4b5fc2952
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 28151c0d0a1641bf938a7672c500e01d
+    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
+  - remote: buf.build
+    owner: protocolbuffers
+    repository: wellknowntypes
+    commit: 657250e6a39648cbb169d079a60bd9ba
+    digest: shake256:00de25001b8dd2e29d85fc4bcc3ede7aed886d76d67f5e0f7a9b320b90f871d3eb73507d50818d823a0512f3f8db77a11c043685528403e31ff3fef18323a9fb

package/src/protod/test-data/buf.yaml

@@ -0,0 +1,25 @@
+# This module represents buf.build/cosmos/cosmos-sdk
+version: v1
+name: buf.build/cosmos/cosmos-sdk
+deps:
+  - buf.build/cosmos/cosmos-proto
+  - buf.build/cosmos/gogo-proto
+  - buf.build/googleapis/googleapis
+  - buf.build/protocolbuffers/wellknowntypes:v23.4
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+    - FILE_LOWER_SNAKE_CASE
+  except:
+    - UNARY_RPC
+    - COMMENT_FIELD
+    - SERVICE_SUFFIX
+    - PACKAGE_VERSION_SUFFIX
+    - RPC_REQUEST_STANDARD_NAME
+    - ENUM_NO_ALLOW_ALIAS
+  ignore:
+    - tendermint

package/src/protod/test-data/buf2.yaml

@@ -0,0 +1,20 @@
+# This module represents buf.build/cosmos/cosmos-sdk
+version: v1
+name: buf.build/cosmos/cosmos-sdk
+breaking:
+  use:
+    - FILE
+lint:
+  use:
+    - DEFAULT
+    - COMMENTS
+    - FILE_LOWER_SNAKE_CASE
+  except:
+    - UNARY_RPC
+    - COMMENT_FIELD
+    - SERVICE_SUFFIX
+    - PACKAGE_VERSION_SUFFIX
+    - RPC_REQUEST_STANDARD_NAME
+    - ENUM_NO_ALLOW_ALIAS
+  ignore:
+    - tendermint

package/src/protod/test-data/cosmos/buf.lock

@@ -0,0 +1,23 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: cosmos
+    repository: cosmos-proto
+    commit: 1935555c206d4afb9e94615dfd0fad31
+    digest: shake256:c74d91a3ac7ae07d579e90eee33abf9b29664047ac8816500cf22c081fec0d72d62c89ce0bebafc1f6fec7aa5315be72606717740ca95007248425102c365377
+  - remote: buf.build
+    owner: cosmos
+    repository: gogo-proto
+    commit: 5e5b9fdd01804356895f8f79a6f1ddc1
+    digest: shake256:0b85da49e2e5f9ebc4806eae058e2f56096ff3b1c59d1fb7c190413dd15f45dd456f0b69ced9059341c80795d2b6c943de15b120a9e0308b499e43e4b5fc2952
+  - remote: buf.build
+    owner: googleapis
+    repository: googleapis
+    commit: 28151c0d0a1641bf938a7672c500e01d
+    digest: shake256:49215edf8ef57f7863004539deff8834cfb2195113f0b890dd1f67815d9353e28e668019165b9d872395871eeafcbab3ccfdb2b5f11734d3cca95be9e8d139de
+  - remote: buf.build
+    owner: protocolbuffers
+    repository: wellknowntypes
+    commit: 657250e6a39648cbb169d079a60bd9ba
+    digest: shake256:00de25001b8dd2e29d85fc4bcc3ede7aed886d76d67f5e0f7a9b320b90f871d3eb73507d50818d823a0512f3f8db77a11c043685528403e31ff3fef18323a9fb

package/src/protod/test-data/cosmos/crypto/secp256k1/keys.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+package cosmos.crypto.secp256k1;
+
+import "amino/amino.proto";
+import "gogoproto/gogo.proto";
+
+option go_package = "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1";
+
+// PubKey defines a secp256k1 public key
+// Key is the compressed form of the pubkey. The first byte depends is a 0x02 byte
+// if the y-coordinate is the lexicographically largest of the two associated with
+// the x-coordinate. Otherwise the first byte is a 0x03.
+// This prefix is followed with the x-coordinate.
+message PubKey {
+  option (amino.name) = "tendermint/PubKeySecp256k1";
+  // The Amino encoding is simply the inner bytes field, and not the Amino
+  // encoding of the whole PubKey struct.
+  //
+  // Example (JSON):
+  // s := PubKey{Key: []byte{0x01}}
+  // out := AminoJSONEncoder(s)
+  //
+  // Then we have:
+  // out == `"MQ=="`
+  // out != `{"key":"MQ=="}`
+  option (amino.message_encoding)     = "key_field";
+  option (gogoproto.goproto_stringer) = false;
+
+  bytes key = 1;
+}
+
+// PrivKey defines a secp256k1 private key.
+message PrivKey {
+  option (amino.name)             = "tendermint/PrivKeySecp256k1";
+  option (amino.message_encoding) = "key_field";
+
+  bytes key = 1;
+}

package/src/protod/test-data/model.proto

@@ -0,0 +1,101 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.ai.generativelanguage.v1beta;
+
+import "google/api/field_behavior.proto";
+import "google/api/resource.proto";
+
+option go_package = "cloud.google.com/go/ai/generativelanguage/apiv1beta/generativelanguagepb;generativelanguagepb";
+option java_multiple_files = true;
+option java_outer_classname = "ModelProto";
+option java_package = "com.google.ai.generativelanguage.v1beta";
+
+// Information about a Generative Language Model.
+message Model {
+  option (google.api.resource) = {
+    type: "generativelanguage.googleapis.com/Model"
+    pattern: "models/{model}"
+  };
+
+  // Required. The resource name of the `Model`.
+  //
+  // Format: `models/{model}` with a `{model}` naming convention of:
+  //
+  // * "{base_model_id}-{version}"
+  //
+  // Examples:
+  //
+  // * `models/chat-bison-001`
+  string name = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. The name of the base model, pass this to the generation request.
+  //
+  // Examples:
+  //
+  // * `chat-bison`
+  string base_model_id = 2 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. The version number of the model.
+  //
+  // This represents the major version
+  string version = 3 [(google.api.field_behavior) = REQUIRED];
+
+  // The human-readable name of the model. E.g. "Chat Bison".
+  //
+  // The name can be up to 128 characters long and can consist of any UTF-8
+  // characters.
+  string display_name = 4;
+
+  // A short description of the model.
+  string description = 5;
+
+  // Maximum number of input tokens allowed for this model.
+  int32 input_token_limit = 6;
+
+  // Maximum number of output tokens available for this model.
+  int32 output_token_limit = 7;
+
+  // The model's supported generation methods.
+  //
+  // The method names are defined as Pascal case
+  // strings, such as `generateMessage` which correspond to API methods.
+  repeated string supported_generation_methods = 8;
+
+  // Controls the randomness of the output.
+  //
+  // Values can range over `[0.0,1.0]`, inclusive. A value closer to `1.0` will
+  // produce responses that are more varied, while a value closer to `0.0` will
+  // typically result in less surprising responses from the model.
+  // This value specifies default to be used by the backend while making the
+  // call to the model.
+  optional float temperature = 9;
+
+  // For Nucleus sampling.
+  //
+  // Nucleus sampling considers the smallest set of tokens whose probability
+  // sum is at least `top_p`.
+  // This value specifies default to be used by the backend while making the
+  // call to the model.
+  optional float top_p = 10;
+
+  // For Top-k sampling.
+  //
+  // Top-k sampling considers the set of `top_k` most probable tokens.
+  // This value specifies default to be used by the backend while making the
+  // call to the model.
+  optional int32 top_k = 11;
+}

package/src/protod/types.ts

@@ -0,0 +1,55 @@
+export interface Repo {
+  owner: string;
+  repo: string;
+}
+
+export interface BufInfo extends Repo {
+  git: Repo[];
+}
+
+export interface GitInfo extends Repo {
+  branch: string;
+  protoDir?: string;
+  protoPath?: string;
+}
+
+export interface DownloadOptions {
+  repos?: {
+    owner: string;
+    repo: string;
+    branch?: string;
+  }[];
+  protoDirMapping?: Record<string, string>;
+  outDir?: string;
+  ssh?: boolean;
+  targets?: string[];
+  tempRepoDir?: string;
+  deleteTempRepoDir?: boolean;
+}
+
+export interface CloneAllOptions {
+  repos?: {
+    owner: string;
+    repo: string;
+    branch?: string;
+  }[];
+  protoDirMapping?: Record<string, string>;
+  gitModulesDir: string;
+  ssh: boolean;
+}
+
+export interface CloneOptions {
+  owner: string;
+  repo: string;
+  branch?: string;
+  protoDirMapping?: Record<string, string>;
+  gitModulesDir: string;
+  ssh: boolean;
+  cloned?: Record<string, GitInfo>;
+}
+
+export interface ProtoCopyOptions {
+  sources: Record<string, GitInfo>;
+  targets: string[];
+  outDir: string;
+}