@taskeren/bungie-api-ts 5.10.0

package/generator/generate-common.ts

@@ -0,0 +1,123 @@
+import { DefInfo, getReferencedTypes, getRef } from './util.js';
+import _ from 'lodash';
+import path from 'path';
+import fs from 'fs';
+import mkdirp from 'mkdirp';
+import { OpenAPIObject, SchemaObject, ReferenceObject } from 'openapi3-ts';
+
+export function generateHeader(doc: OpenAPIObject): string {
+  const { info } = doc;
+  return `/**
+ * ${info.title}
+ * ${info.description}
+ *
+ * OpenAPI spec version: ${info.version}
+ * Contact: ${info.contact!.email}
+ *
+ * NOTE: This class is auto generated by the bungie-api-ts code generator program.
+ * https://github.com/DestinyItemManager/bungie-api-ts
+ * Do not edit these files manually.
+ */`;
+}
+
+/**
+ * Add the name of a referenced type from `schema` to the list of symbols to
+ * import for a given imported module file.
+ */
+export function addImport(
+  doc: OpenAPIObject,
+  schema: SchemaObject | ReferenceObject,
+  componentByDef: Readonly<{ [def: string]: DefInfo }>,
+  importFiles: { [filename: string]: Set<string> } // mutated
+) {
+  const typeRef = getReferencedTypes(schema);
+  if (typeRef && componentByDef[typeRef]) {
+    if (typeRef.includes('/responses/')) {
+      const component = getRef(doc, typeRef);
+      if (component) {
+        const property = component.properties!.Response;
+        if (property) {
+          (importFiles['common.ts'] ??= new Set()).add('ServerResponse');
+          addImport(doc, property, componentByDef, importFiles);
+          return;
+        }
+      }
+    }
+
+    const filename = componentByDef[typeRef].filename;
+    (importFiles[filename] ??= new Set()).add(componentByDef[typeRef].interfaceName);
+  }
+}
+
+/**
+ * Given an output filename and a list of all symbols to be imported, generate
+ * the actual import lines.
+ */
+export function generateImports(
+  filename: string,
+  importFiles: Readonly<{ [filename: string]: Set<string> }>
+): string {
+  return _.compact(
+    _.map(importFiles, (types, f) => {
+      const absImport = path.resolve('generated-src', f);
+      const absDest = path.resolve(filename);
+      if (absImport === absDest) {
+        return undefined;
+      }
+      let relativePath = path.relative(path.dirname(absDest), absImport).replace(/(\.d)?\.ts$/, '');
+      if (!relativePath.startsWith('.')) {
+        relativePath = './' + relativePath;
+      }
+      if (path.sep === '\\') relativePath = relativePath.replace(/\\/g, '/');
+      return `import {
+  ${[...types].sort().join(',\n  ')}
+} from '${relativePath}.js';`;
+    })
+  )
+    .sort()
+    .join('\n');
+}
+
+/**
+ * Print a JSDoc documentation comment with the given string as its contents.
+ */
+export function docComment(text: string) {
+  const lines = _.flatten(
+    text
+      .trim()
+      .split('\n')
+      .map((l) => l.replace(/(.{1,80}(?:\W|$))/g, '$1\n').split('\n'))
+  ).map((s: string) => s.trim());
+  lines.pop();
+
+  if (lines.length === 1) {
+    return `/** ${lines} */`;
+  }
+
+  return `/**
+${lines.map((line) => (line.length ? ' * ' + line : ' *')).join('\n')}
+ */`;
+}
+
+/**
+ * Indent the given text a certain number of levels.
+ */
+export function indent(text: string, indentLevel: number) {
+  const lines = text.split('\n');
+  return lines.map((line) => '  '.repeat(indentLevel) + line).join('\n');
+}
+
+/**
+ * Write a string contents out to a file.
+ */
+export async function writeOutFile(filename: string, contents: string) {
+  try {
+    await mkdirp(path.dirname(filename));
+
+    fs.writeFile(filename, contents, null, (error) => {
+      console.log(error ? error : `Done with ${filename}!`);
+    });
+  } catch (e) {
+    console.error(e);
+  }
+}

package/generator/generate-index.ts

@@ -0,0 +1,45 @@
+import { OpenAPIObject } from 'openapi3-ts';
+import { generateHeader, writeOutFile } from './generate-common.js';
+import { DefInfo } from './util.js';
+
+export function generateIndex(
+  tag: string,
+  doc: OpenAPIObject,
+  componentsByFile: {
+    [filename: string]: DefInfo[];
+  }
+) {
+  const filename = `generated-src/${tag.toLowerCase()}/index.ts`;
+
+  let imports = `export * from '../common.js';${
+    tag !== 'Destiny2' ? "\nexport * from '../platform.js';" : ''
+  }
+export type {HttpClientConfig, HttpClient} from '../http.js';
+export * from './api.js';`;
+
+  if (componentsByFile[`${tag.toLowerCase()}/interfaces.ts`]) {
+    imports = `${imports}\nexport * from './interfaces.js';`;
+  }
+
+  // Destiny2 service has special manifest helpers
+  if (tag === 'Destiny2') {
+    imports = `${imports}\nexport * from './manifest.js';`;
+  }
+
+  const definition = [generateHeader(doc), imports].join('\n\n') + '\n';
+
+  writeOutFile(filename, definition);
+}
+
+export function generateSuperIndex(tags: string[], doc: OpenAPIObject) {
+  const filename = `generated-src/index.ts`;
+
+  const imports = tags
+    .map((tag) => `import * as ${tag}Import from './${tag.toLowerCase()}/index.js';`)
+    .join('\n');
+  const exportStatements = tags.map((tag) => `export const ${tag} = ${tag}Import;`).join('\n');
+
+  const definition = [generateHeader(doc), imports, exportStatements].join('\n\n') + '\n';
+
+  writeOutFile(filename, definition);
+}

package/generator/generate-interfaces.ts

@@ -0,0 +1,173 @@
+import _ from 'lodash';
+import { DefInfo, getRef, resolveSchemaType } from './util.js';
+import { OpenAPIObject, SchemaObject } from 'openapi3-ts';
+import {
+  generateHeader,
+  generateImports,
+  docComment,
+  indent,
+  addImport,
+  writeOutFile,
+} from './generate-common.js';
+import { missingPropsByInterfaceName } from './missing-props.js';
+
+/**
+ * Some properties aren't marked as nullable in the openapi docs, but they are frequently null in practice and cause trouble.
+ * Adding a property to this list forces it to be emitted as nullable.
+ */
+const frequentlyNullProperties = ['itemCategoryHashes', 'itemValueVisibility'];
+
+export function generateInterfaceDefinitions(
+  file: string,
+  components: DefInfo[],
+  doc: OpenAPIObject,
+  componentByDef: { [def: string]: DefInfo }
+) {
+  const importFiles: { [filename: string]: Set<string> } = {};
+
+  const componentDefinitions = components.map((component) =>
+    generateComponentDefinition(component, doc, componentByDef, importFiles)
+  );
+
+  const filename = `generated-src/${file}`;
+
+  let specialDefinitions;
+  if (file === 'destiny2/interfaces.ts') {
+    specialDefinitions = generateSpecialDefinitions();
+  }
+  if (file === 'common.ts') {
+    specialDefinitions = generateServerResponseDefinitions();
+  }
+
+  const imports = generateImports(filename, importFiles);
+
+  const definition =
+    _.compact([generateHeader(doc), imports, specialDefinitions, ...componentDefinitions]).join(
+      '\n\n'
+    ) + '\n';
+
+  writeOutFile(filename, definition);
+}
+
+function generateComponentDefinition(
+  defInfo: DefInfo,
+  doc: OpenAPIObject,
+  // mutated
+  componentByDef: { [def: string]: DefInfo },
+  // mutated
+  importFiles: { [filename: string]: Set<string> }
+) {
+  const component = getRef(doc, defInfo.def);
+  if (!component) {
+    return undefined;
+  }
+
+  if (component.enum) {
+    return generateEnum(defInfo, component);
+  } else if (isSpecialType(defInfo.interfaceName)) {
+    return undefined;
+  } else {
+    return generateInterfaceSchema(
+      defInfo.interfaceName,
+      component,
+      doc,
+      componentByDef,
+      importFiles
+    );
+  }
+}
+
+function generateEnum(defInfo: DefInfo, component: SchemaObject) {
+  const values = component['x-enum-values']
+    .map((value: SchemaObject) => {
+      const doc = value.description ? docComment(value.description) + '\n' : '';
+      return `${doc}${value.identifier} = ${value.numericValue}`;
+    })
+    .join(',\n');
+
+  const docs = component.description ? [component.description] : [];
+  if (component['x-enum-is-bitmask']) {
+    docs.push(
+      `This enum represents a set of flags - use bitwise operators to check which of these match your value.`
+    );
+  }
+
+  const docString = docs.length ? docComment(docs.join('\n')) + '\n' : '';
+
+  // TODO: const enums are super efficient (they get inlined) but we may want to change this if we want to do things like
+  // print out the name of an enum case.
+  return `${docString}export const enum ${defInfo.interfaceName} {
+${indent(values, 1)}
+}`;
+}
+
+function generateInterfaceSchema(
+  interfaceName: string,
+  component: SchemaObject,
+  doc: OpenAPIObject,
+  componentByDef: { [def: string]: DefInfo },
+  importFiles: { [filename: string]: Set<string> }
+) {
+  // for *missing* props, this acts conservatively and only inserts our overrides if the prop isn't provided by bungie at all.
+  // we could add *overrides* later but for now don't want to overwrite a prop if they get around to adding it.
+  const missingProps = missingPropsByInterfaceName[interfaceName] ?? {};
+  for (const k in component.properties) delete missingProps[k];
+  const properties = { ...component.properties, ...missingProps };
+
+  const parameterArgs = _.map(properties, (schema: SchemaObject, param) => {
+    const paramType = resolveSchemaType(schema, doc);
+    addImport(doc, schema, componentByDef, importFiles);
+    const docs = schema.description ? [schema.description] : [];
+
+    const crossMappedDef = schema['x-mapped-definition']?.$ref;
+    if (crossMappedDef) {
+      docs.push(`Mapped to ${componentByDef[crossMappedDef].interfaceName} in the manifest.`);
+    }
+
+    if (schema['x-enum-is-bitmask']) {
+      docs.push(
+        `This enum represents a set of flags - use bitwise operators to check which of these match your value.`
+      );
+    }
+    const docString = docs.length ? docComment(docs.join('\n')) + '\n' : '';
+
+    const isNullable =
+      schema.nullable ||
+      frequentlyNullProperties.includes(param) ||
+      schema.description?.toLowerCase().includes('null');
+
+    return `${docString}readonly ${param}${isNullable ? '?' : ''}: ${paramType};`;
+  });
+
+  const docString = component.description ? docComment(component.description) + '\n' : '';
+  return `${docString}export interface ${interfaceName} {
+${indent(parameterArgs.join('\n'), 1)}
+}`;
+}
+
+function isSpecialType(name: string) {
+  return name.includes('>');
+}
+
+function generateSpecialDefinitions() {
+  return `export interface SingleComponentResponse<T> {
+  readonly data?: T;
+  readonly privacy: ComponentPrivacySetting;
+}
+
+export interface DictionaryComponentResponse<T> {
+  readonly data?: { [key: string]: T };
+  readonly privacy: ComponentPrivacySetting;
+}`;
+}
+
+function generateServerResponseDefinitions() {
+  return `export interface ServerResponse<T> {
+  readonly Response: T;
+  readonly ErrorCode: PlatformErrorCodes;
+  readonly ThrottleSeconds: number;
+  readonly ErrorStatus: string;
+  readonly Message: string;
+  readonly MessageData: { [key: string]: string };
+}`;
+}

package/generator/generate-manifest.ts

@@ -0,0 +1,217 @@
+import { DefInfo } from './util.js';
+import { OpenAPIObject } from 'openapi3-ts';
+import { writeOutFile } from './generate-common.js';
+
+const httpClientType = `import { HttpClient, get } from '../http.js';`;
+
+const manifestMetadataPromise = (async () => {
+  try {
+    let manifestMeta = await (fetch as any)(
+      'https://www.bungie.net/Platform/Destiny2/Manifest/'
+    ).then((res: Response) => res.json());
+    return (manifestMeta as any).Response;
+  } catch (e) {
+    console.log(e);
+    process.exit(1);
+  }
+})();
+
+export async function generateManifestUtils(components: DefInfo[], doc: OpenAPIObject) {
+  const manifestStructure = await generateManifestDefinitions(components);
+
+  const filename = `generated-src/destiny2/manifest.ts`;
+
+  const definition =
+    [generateManifestHeader(doc), httpClientType, manifestStructure, manifestUtilDefinitions].join(
+      '\n\n'
+    ) + '\n';
+
+  writeOutFile(filename, definition);
+}
+
+async function generateManifestDefinitions(components: DefInfo[]) {
+  let manifestMetadata = await manifestMetadataPromise;
+
+  // defs we have documentation for. some stuff in manifest doesn't have interface definitions. idk why.
+  const documentedDefs = components.map((component) => component.interfaceName);
+
+  // exclude some tables from the definitionmanifest table because we don't have the format for them
+  const defsToInclude = Object.keys(manifestMetadata.jsonWorldComponentContentPaths.en).filter(
+    (tableName) => documentedDefs.includes(tableName)
+  );
+
+  const languageList = Object.keys(manifestMetadata.jsonWorldComponentContentPaths).sort();
+
+  return `import {${defsToInclude.map((c) => `\n  ${c},`).join('')}
+  DestinyManifest
+} from './interfaces.js';
+
+/**
+ * this describes a big object holding several tables of hash-keyed DestinyDefinitions.
+ * this is roughly what you get if you decode the gigantic, single-json manifest blob,
+ * but also just what we use here to dole out single-table, typed definitions
+ */
+export interface AllDestinyManifestComponents {
+${defsToInclude
+  .map((manifestComponent) => `  ${manifestComponent}: { [key: number]: ${manifestComponent} };\n`)
+  .join('')}}
+
+export type DestinyInventoryItemLiteDefinition = Omit<
+  DestinyInventoryItemDefinition,
+  | "iconWatermarkShelved" | "screenshot" | "flavorText" | "tooltipStyle"
+  | "action" | "stats" | "statsEnabledUnlockExpression" | "translationBlock"
+  | "preview" | "sourceData" | "objectives" | "metrics" | "plug"
+  | "gearset" | "sack" | "sockets" | "investmentStats" | "traitIds"
+  | "traitHashes" | "boundToRelease" | "hash" | "index" | "contentIdentifier"
+  | "redacted" | "blacklisted"
+>;
+
+/**
+ * languages the manifest comes in, as their required keys to download them
+ */
+export const destinyManifestLanguages = [
+${languageList.map((l) => `  '${l}',`).join('\n')}
+] as const;
+
+export type DestinyManifestLanguage = typeof destinyManifestLanguages[number];
+
+`;
+}
+
+function generateManifestHeader(doc: OpenAPIObject): string {
+  const { info } = doc;
+  return `/**
+ * these helper functions and definitions are based off the structure of DestinyManifest
+ * in the bungie.net API spec, but are not explicity defined endpoints in the spec.
+ *
+ * they were last hand-checked for OpenAPI spec version 2.8.0,
+ * and have been automatically tested for the latest OpenAPI spec version ${info.version}.
+ * if there are typing issues with them, please let us know at the below repo.
+ *
+ * NOTE: This class is auto generated by the bungie-api-ts code generator program.
+ * https://github.com/DestinyItemManager/bungie-api-ts
+ * Do not edit these files manually.
+ */`;
+}
+
+const manifestUtilDefinitions = `
+// thoughts:
+// this relies on the assumption that the separate
+// manifest pieces offered in jsonWorldComponentContentPaths,
+// will all be present in the big manifest at jsonWorldContentPaths.
+
+// this has been the case so far, but there aren't
+// strict spec standards for how the manifest will be available
+
+export type DestinyManifestComponentName = keyof AllDestinyManifestComponents;
+
+export type DestinyManifestSlice<K extends Readonly<DestinyManifestComponentName[]>> = Pick<
+  AllDestinyManifestComponents,
+  K[number]
+>;
+
+/**
+ * given a STRING table name, returns that TYPE, so that you can write a function like:
+ * func<K extends DestinyManifestComponentName>(arg0:K):DestinyDefinitionFrom<K>{...}
+ * i.e.
+ * func('DestinyInventoryItemDefinition') will return type DestinyInventoryItemDefinition
+ */
+export type DestinyDefinitionFrom<
+  K extends DestinyManifestComponentName
+> = AllDestinyManifestComponents[K][number];
+
+export interface GetAllDestinyManifestComponentsParams {
+  destinyManifest: DestinyManifest;
+  language: DestinyManifestLanguage;
+}
+/** fetches the enormous combined JSON manifest file */
+export function getAllDestinyManifestComponents(
+  http: HttpClient,
+  params: GetAllDestinyManifestComponentsParams
+): Promise<AllDestinyManifestComponents> {
+  return get(http, 'https://www.bungie.net'+params.destinyManifest.jsonWorldContentPaths[params.language]);
+}
+
+export interface GetDestinyManifestComponentParams<T extends DestinyManifestComponentName> {
+  destinyManifest: DestinyManifest;
+  tableName: T;
+  language: DestinyManifestLanguage;
+}
+/**
+ * this fetches and returns a single table (Component) from the d2 manifest
+ * i.e. DestinyInventoryItemDefinition / DestinyObjectiveDefinition /
+ * DestinyVendorDefinition / DestinySeasonDefinition / etc.
+ *
+ * due to typescript limitations, the table name needs to be recognized by
+ * typescript as a const (not mutable between inception and going into the function),
+ * so that it considers it a table name and not just a string.
+ *
+ * this is easy with a string, since
+ *
+ * \`const x = 'thing';\` is type \`'thing'\`, not type \`string\`,
+ *
+ * but make sure it's not a \`let x =\` or a dynamically set string.
+ */
+ export async function getDestinyManifestComponent<T extends DestinyManifestComponentName>(
+  http: HttpClient,
+  params: GetDestinyManifestComponentParams<T>
+): Promise<AllDestinyManifestComponents[T]> {
+  const url = 'https://www.bungie.net' +
+      params.destinyManifest.jsonWorldComponentContentPaths[params.language][params.tableName];
+  try {
+    return await get(http, url);
+  } catch (e) {
+    try {
+      return await get(http, \`\${url}?retry\`);
+    } catch {
+      throw e;
+    }
+  }
+}
+
+export interface GetDestinyManifestSliceParams<T extends DestinyManifestComponentName[]> {
+  destinyManifest: DestinyManifest;
+  tableNames: T;
+  language: DestinyManifestLanguage;
+}
+/**
+ * this returns a similar structure to getAllDestinyManifestComponents (the big manifest json)
+ * but only specific components within. it bundles multiple single tables requests,
+ * into a single properly typed object with keys named after manifest components
+ *
+ * i.e. \`{ DestinyInventoryItemDefinition: etc...,
+ * DestinyObjectiveDefinition: etc... }\`
+ *
+ * due to typescript limitations, the array of tableNames needs to be recognized by
+ * typescript as readonly (not mutable between inception and going into the function),
+ * so that it considers them table names and not just strings.
+ *
+ * like \`['DestinyInventoryItemDefinition' as const]\`
+ *
+ * or maybe \`['DestinyInventoryItemDefinition'] as const\`
+ *
+ * or just feed in into the function hardcoded like
+ *
+ * \`function(['DestinyInventoryItemDefinition'])\`
+ */
+export async function getDestinyManifestSlice<T extends DestinyManifestComponentName[]>(
+  http: HttpClient,
+  params: GetDestinyManifestSliceParams<T>
+): Promise<DestinyManifestSlice<T>> {
+  const downloadedTables = await Promise.all(
+    params.tableNames.map(async (tableName) => {
+      const tableContent = await getDestinyManifestComponent(http, {
+        destinyManifest: params.destinyManifest,
+        tableName,
+        language: params.language,
+      });
+      return { tableName, tableContent };
+    })
+  );
+  const manifestSlice = {} as AllDestinyManifestComponents;
+  for (const downloadedTable of downloadedTables) {
+    manifestSlice[downloadedTable.tableName] = downloadedTable.tableContent as any;
+  }
+  return manifestSlice as DestinyManifestSlice<T>;
+}
+`;

package/generator/generate-package-json.ts

@@ -0,0 +1,45 @@
+import packageJson from './package.json' with { type: 'json' };
+import { writeOutFile } from './generate-common.js';
+
+/**
+ * Generate a package.json file for the library, based on the package.json file at the
+ * root of the repo.
+ */
+export function generatePackageJson(modules: string[]) {
+  console.log('Generate package.json??');
+  const moduleExports: { [key: string]: any } = {
+    '.': {
+      default: './index.js',
+      types: './index.d.ts',
+    },
+    './http': {
+      default: './http.js',
+      types: './http.d.ts',
+    },
+    './package.json': './package.json',
+  };
+
+  for (const module of modules) {
+    moduleExports[`./${module.toLowerCase()}`] = {
+      default: `./${module.toLowerCase()}/index.js`,
+      types: `./${module.toLowerCase()}/index.d.ts`,
+    };
+  }
+
+  // clear out dependencies
+  const newPackageJson = {
+    ...packageJson,
+    main: './index.js',
+    types: './index.d.ts',
+    module: './index.js',
+    exports: moduleExports,
+    scripts: {},
+    dependencies: {},
+    devDependencies: {},
+  };
+
+  writeOutFile(
+    'generated-src/package.json.notyet',
+    JSON.stringify(newPackageJson, undefined, '  ')
+  );
+}

package/generator/generate.ts

@@ -0,0 +1,54 @@
+/**
+ * This is a custom, hand-rolled generator for TS typings for the Bungie.net API. It's meant for use
+ * in DIM, but is free for anyone to use.
+ */
+
+import fs from 'fs';
+import _ from 'lodash';
+import { OpenAPIObject, PathItemObject } from 'openapi3-ts';
+import { generateIndex, generateSuperIndex } from './generate-index.js';
+
+import { DefInfo } from './util.js';
+import { computeTypeMaps } from './type-index.js';
+import { generateInterfaceDefinitions } from './generate-interfaces.js';
+import { generateManifestUtils } from './generate-manifest.js';
+import { generateServiceDefinition } from './generate-api.js';
+import { generatePackageJson } from './generate-package-json.js';
+
+// allow some async operations
+(async () => {
+  const doc = JSON.parse(fs.readFileSync('api-src/openapi.json').toString()) as OpenAPIObject;
+
+  // Pairs of [request path, path service description]
+  const pathPairs = _.toPairs(doc.paths) as [string, PathItemObject][];
+
+  // Grouped by "tag" which says which service (destiny, groups, forums, etc)
+  const pathPairsByTag = _.groupBy(pathPairs, ([path, desc]) => {
+    return (desc.get || desc.post)!.tags![0];
+  });
+  pathPairsByTag['Core'] = pathPairsByTag[''];
+  delete pathPairsByTag[''];
+
+  const { componentsByFile, componentByDef } = computeTypeMaps(pathPairsByTag, doc);
+
+  _.each(componentsByFile, (components: DefInfo[], file: string) => {
+    generateInterfaceDefinitions(file, components, doc, componentByDef);
+  });
+
+  await generateManifestUtils(componentsByFile['destiny2/interfaces.ts'], doc);
+
+  _.each(pathPairsByTag, (paths, tag) => {
+    generateServiceDefinition(tag, paths, doc, componentByDef);
+  });
+
+  _.each(pathPairsByTag, (paths, tag) => {
+    generateIndex(tag, doc, componentsByFile);
+  });
+
+  generateSuperIndex(Object.keys(pathPairsByTag), doc);
+
+  // read top package.json, remove dependencies, add exports block per service
+  generatePackageJson(Object.keys(pathPairsByTag));
+
+  // some way to mark "preview" stuff
+})();