@ohos-ports/curlconverter 4.12.0-beta.0

package/package.json

@@ -0,0 +1,97 @@
+{
+  "name": "@ohos-ports/curlconverter",
+  "version": "4.12.0-beta.0",
+  "description": "convert curl commands to Python, JavaScript, Go, PHP and more",
+  "homepage": "https://curlconverter.com",
+  "author": {
+    "name": "Nick Carneiro",
+    "email": "nickc@trillworks.com",
+    "url": "https://trillworks.com"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ohos-ports/ohos-ports.git",
+    "directory": "ports/curlconverter/4.12.0"
+  },
+  "license": "MIT",
+  "keywords": [
+    "curl",
+    "http",
+    "requests",
+    "python",
+    "converter"
+  ],
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "type": "module",
+  "dependencies": {
+    "jsesc": "^3.0.2",
+    "lossless-json": "^4.0.2",
+    "@ohos-ports/tree-sitter": "^0.25.2-beta.0",
+    "web-tree-sitter": "^0.24.3",
+    "yamljs": "^0.3.0",
+    "@ohos-ports/tree-sitter-bash": "^0.23.3-beta.0"
+  },
+  "devDependencies": {
+    "@eslint/eslintrc": "^3.1.0",
+    "@eslint/js": "^9.12.0",
+    "@types/diff": "^5.2.3",
+    "@types/har-format": "^1.2.16",
+    "@types/jsesc": "^3.0.3",
+    "@types/nunjucks": "^3.2.6",
+    "@types/tape": "^5.6.4",
+    "@types/yamljs": "^0.2.34",
+    "@types/yargs": "^17.0.33",
+    "@typescript-eslint/eslint-plugin": "^8.9.0",
+    "@typescript-eslint/parser": "^8.9.0",
+    "c8": "^10.1.2",
+    "colors": "^1.4.0",
+    "cross-env": "^7.0.3",
+    "diff": "^7.0.0",
+    "eslint": "^9.12.0",
+    "eslint-config-prettier": "^9.1.0",
+    "glob": "^11.0.0",
+    "husky": "^9.1.6",
+    "lint-staged": "^15.2.10",
+    "prettier": "^3.3.3",
+    "tape": "^5.9.0",
+    "@ohos-ports/tree-sitter-cli": "^0.27.0-beta.0",
+    "typescript": "^5.6.3",
+    "yargs": "^17.7.2"
+  },
+  "scripts": {
+    "compile": "tsc",
+    "start": "tsc --watch --preserveWatchOutput -p ./",
+    "pretest": "cross-env TSC_COMPILE_ON_ERROR=true npm run compile || exit 0",
+    "test": "cross-env NODE_OPTIONS=\"--enable-source-maps --unhandled-rejections=strict\" npx tape dist/test/test.js",
+    "coverage": "c8 --check-coverage npm run test",
+    "prelint": "npm run compile",
+    "lint": "eslint src",
+    "format": "prettier --write .",
+    "prefix": "npm run compile",
+    "fix": "eslint src --fix",
+    "prepare": "npx husky && npm run compile && npx tree-sitter build --wasm node_modules/tree-sitter-bash --output dist/tree-sitter-bash.wasm",
+    "pregen-test": "npm run compile",
+    "gen-test": "node --enable-source-maps ./dist/tools/gen-test.js",
+    "compare-requests": "npm run compile && node --enable-source-maps ./dist/tools/compare-requests.js",
+    "compare-request": "npm run compile && node --enable-source-maps ./dist/tools/compare-requests.js"
+  },
+  "lint-staged": {
+    "**/*": "prettier --write --ignore-unknown"
+  },
+  "bin": "dist/src/cli.js",
+  "browser": {
+    "./dist/src/shell/Parser.js": "./dist/src/shell/webParser.js"
+  },
+  "bugs": {
+    "url": "https://github.com/ohos-ports/ohos-ports/issues"
+  },
+  "files": [
+    "dist",
+    "src",
+    "tools",
+    "index.d.ts",
+    "README.md",
+    "LICENSE"
+  ]
+}

package/src/Headers.ts

@@ -0,0 +1,334 @@
+import { Word, eq, joinWords } from "./shell/Word.js";
+import type { Warnings } from "./Warnings.js";
+
+// https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Standard_request_fields
+// and then searched for "#" in the RFCs that define each header
+const COMMA_SEPARATED = new Set(
+  [
+    "A-IM",
+    "Accept",
+    "Accept-Charset",
+    // "Accept-Datetime",
+    "Accept-Encoding",
+    "Accept-Language",
+    // "Access-Control-Request-Method",
+    "Access-Control-Request-Headers",
+    // TODO: auth-scheme [ 1*SP ( token68 / #auth-param ) ]
+    // "Authorization",
+    "Cache-Control",
+    "Connection",
+    "Content-Encoding",
+    // "Content-Length",
+    // "Content-MD5",
+    // "Content-Type", // semicolon
+    // "Cookie", // semicolon
+    // "Date",
+    "Expect",
+    "Forwarded",
+    // "From",
+    // "Host",
+    // "HTTP2-Settings",
+    "If-Match",
+    // "If-Modified-Since",
+    "If-None-Match",
+    // "If-Range",
+    // "If-Unmodified-Since",
+    // "Max-Forwards",
+    // "Origin",
+    // "Pragma",
+    // "Prefer", // semicolon
+    // "Proxy-Authorization",
+    "Range",
+    // "Referer",
+    "TE",
+    "Trailer",
+    "Transfer-Encoding",
+    // "User-Agent",
+    "Upgrade",
+    "Via",
+    "Warning",
+  ].map((h) => h.toLowerCase()),
+);
+const SEMICOLON_SEPARATED = new Set(
+  ["Content-Type", "Cookie", "Prefer"].map((h) => h.toLowerCase()),
+);
+
+export class Headers implements Iterable<[Word, Word | null]> {
+  // null means the header must not be sent, for example
+  // curl -H 'Host:' example.com
+  readonly headers: [Word, Word | null][];
+  readonly lowercase: boolean;
+
+  constructor(
+    headerArgs?: Word[],
+    warnings: Warnings = [],
+    // Used for warnings
+    argName = "--header/H",
+  ) {
+    let headers: [Word, Word | null][] = [];
+
+    if (headerArgs) {
+      for (const header of headerArgs) {
+        if (header.startsWith("@")) {
+          warnings.push([
+            "header-file",
+            "passing a file for " +
+              argName +
+              " is not supported: " +
+              JSON.stringify(header.toString()),
+          ]);
+          continue;
+        }
+
+        if (header.includes(":")) {
+          const [name, value] = header.split(":", 2);
+          // Passing -H 'Header-Name:' disables sending of that header.
+          // If the colon is followed by just spaces they're ignored
+          // unless it's -H 'Host: '
+          // https://github.com/curl/curl/issues/12782
+          const hasValue =
+            value && (eq(name, "Host") ? value : value.trim()).toBool();
+          const headerValue = hasValue ? value.removeFirstChar(" ") : null;
+          headers.push([name, headerValue]);
+        } else if (header.includes(";")) {
+          const [name] = header.split(";", 2);
+          headers.push([name, new Word()]);
+        } else {
+          // TODO: warn that this header arg is ignored?
+        }
+      }
+    }
+    this.lowercase =
+      headers.length > 0 && headers.every((h) => eq(h[0], h[0].toLowerCase()));
+
+    // Handle repeated headers
+    // For Cookie and Accept, merge the values using ';' and ',' respectively
+    // For other headers, warn about the repeated header
+    const uniqueHeaders: { [key: string]: [Word, Word | null][] } = {};
+    for (const [name, value] of headers) {
+      // TODO: something better, at least warn that variable is ignored
+      const lowerName = name.toLowerCase().toString();
+      if (!uniqueHeaders[lowerName]) {
+        uniqueHeaders[lowerName] = [];
+      }
+      uniqueHeaders[lowerName].push([name, value]);
+    }
+
+    headers = [];
+    for (const [lowerName, repeatedHeaders] of Object.entries(uniqueHeaders)) {
+      if (repeatedHeaders.length === 1) {
+        headers.push(repeatedHeaders[0]);
+        continue;
+      }
+      // If they're all null, just use the first one
+      if (repeatedHeaders.every((h) => h[1] === null)) {
+        const lastRepeat = repeatedHeaders[repeatedHeaders.length - 1];
+        // Warn users if some are capitalized differently
+        if (new Set(repeatedHeaders.map((h) => h[0])).size > 1) {
+          warnings.push([
+            "repeated-header",
+            `"${lastRepeat[0]}" header unset ${repeatedHeaders.length} times`,
+          ]);
+        }
+        headers.push(lastRepeat);
+        continue;
+      }
+      // Otherwise there's at least one non-null value, so we can ignore the nulls
+      // TODO: if the values of the repeated headers are the same, just use the first one
+      //     'content-type': 'application/json; application/json',
+      // doesn't really make sense
+      const nonEmptyHeaders = repeatedHeaders.filter((h) => h[1] !== null);
+      if (nonEmptyHeaders.length === 1) {
+        headers.push(nonEmptyHeaders[0]);
+        continue;
+      }
+      let mergeChar = "";
+      if (COMMA_SEPARATED.has(lowerName)) {
+        mergeChar = ", ";
+      } else if (SEMICOLON_SEPARATED.has(lowerName)) {
+        mergeChar = "; ";
+      }
+      if (mergeChar) {
+        const merged = joinWords(
+          nonEmptyHeaders.map((h) => h[1]) as Word[],
+          mergeChar,
+        );
+        warnings.push([
+          "repeated-header",
+          `merged ${nonEmptyHeaders.length} "${
+            nonEmptyHeaders[nonEmptyHeaders.length - 1][0]
+          }" headers together with "${mergeChar.trim()}"`,
+        ]);
+        headers.push([nonEmptyHeaders[0][0], merged]);
+        continue;
+      }
+
+      warnings.push([
+        "repeated-header",
+        `found ${nonEmptyHeaders.length} "${
+          nonEmptyHeaders[nonEmptyHeaders.length - 1][0]
+        }" headers, only the last one will be sent`,
+      ]);
+      headers = headers.concat(nonEmptyHeaders);
+    }
+
+    this.headers = headers;
+  }
+
+  get length(): number {
+    return this.headers.length;
+  }
+
+  *[Symbol.iterator](): Iterator<[Word, Word | null]> {
+    for (const h of this.headers) {
+      yield h;
+    }
+  }
+
+  // Gets the first header, matching case-insensitively
+  get(header: string): Word | null | undefined {
+    const lookup = header.toLowerCase();
+    for (const [h, v] of this.headers) {
+      if (h.toLowerCase().toString() === lookup) {
+        return v;
+      }
+    }
+    return undefined;
+  }
+
+  getContentType(): string | null | undefined {
+    const contentTypeHeader = this.get("content-type");
+    if (!contentTypeHeader) {
+      return contentTypeHeader;
+    }
+    return contentTypeHeader.split(";")[0].trim().toString();
+  }
+
+  has(header: Word | string): boolean {
+    const lookup = header.toLowerCase();
+    for (const h of this.headers) {
+      if (eq(h[0].toLowerCase(), lookup)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // Doesn't overwrite existing headers
+  setIfMissing(header: string, value: Word | string): boolean {
+    if (this.has(header)) {
+      return false;
+    }
+
+    if (this.lowercase) {
+      header = header.toLowerCase();
+    }
+    const k = typeof header === "string" ? new Word(header) : header;
+    const v = typeof value === "string" ? new Word(value) : value;
+    this.headers.push([k, v]);
+    return true;
+  }
+
+  prependIfMissing(header: string, value: Word | string) {
+    if (this.has(header)) {
+      return false;
+    }
+
+    if (this.lowercase) {
+      header = header.toLowerCase();
+    }
+    const k = typeof header === "string" ? new Word(header) : header;
+    const v = typeof value === "string" ? new Word(value) : value;
+    this.headers.unshift([k, v]);
+    return true;
+  }
+
+  set(header: string, value: Word | string) {
+    if (this.lowercase) {
+      header = header.toLowerCase();
+    }
+    const k = typeof header === "string" ? new Word(header) : header;
+    const v = typeof value === "string" ? new Word(value) : value;
+
+    // keep it in the same place if we overwrite
+    const searchHeader = k.toLowerCase().toString();
+    for (let i = 0; i < this.headers.length; i++) {
+      if (eq(this.headers[i][0].toLowerCase(), searchHeader)) {
+        this.headers[i][1] = v;
+        return;
+      }
+    }
+
+    this.headers.push([k, v]);
+  }
+
+  delete(header: string) {
+    const lookup = header.toLowerCase();
+    for (let i = this.headers.length - 1; i >= 0; i--) {
+      if (this.headers[i][0].toLowerCase().toString() === lookup) {
+        this.headers.splice(i, 1);
+      }
+    }
+  }
+
+  // TODO: doesn't this skip the next element after deleting?
+  clearNulls() {
+    for (let i = this.headers.length - 1; i >= 0; i--) {
+      if (this.headers[i][1] === null) {
+        this.headers.splice(i, 1);
+      }
+    }
+  }
+
+  // TODO: shouldn't be used
+  count(header: string) {
+    let count = 0;
+    const lookup = header.toLowerCase();
+    for (const h of this.headers || []) {
+      if (h[0].toLowerCase().toString() === lookup) {
+        count += 1;
+      }
+    }
+    return count;
+  }
+
+  toBool(): boolean {
+    return this.headers.length > 0 && this.headers.some((h) => h[1] !== null);
+  }
+}
+
+// Cookie is a type of header.
+export type Cookie = [Word, Word];
+export type Cookies = Array<Cookie>;
+
+export function parseCookiesStrict(cookieString: Word): Cookies | null {
+  const cookies: Cookies = [];
+  for (let cookie of cookieString.split(";")) {
+    cookie = cookie.replace(/^ /, "");
+    const [name, value] = cookie.split("=", 2);
+    if (value === undefined) {
+      return null;
+    }
+    cookies.push([name, value]);
+  }
+  if (new Set(cookies.map((c) => c[0])).size !== cookies.length) {
+    return null;
+  }
+  return cookies;
+}
+
+export function parseCookies(cookieString: Word): Cookies | null {
+  const cookies: Cookies = [];
+  for (let cookie of cookieString.split(";")) {
+    cookie = cookie.trim();
+    if (!cookie) {
+      continue;
+    }
+    const [name, value] = cookie.split("=", 2);
+    cookies.push([name.trim(), (value || "").trim()]);
+  }
+  if (new Set(cookies.map((c) => c[0])).size !== cookies.length) {
+    return null;
+  }
+  return cookies;
+}

package/src/Query.ts

@@ -0,0 +1,147 @@
+import { Word, eq } from "./shell/Word.js";
+import { UTF8encoder } from "./utils.js";
+
+export type QueryList = Array<[Word, Word]>;
+export type QueryDict = Array<[Word, Word | Array<Word>]>;
+
+// Can't have a QueryDict without a QueryList
+export type Query = [QueryList, QueryDict | null] | [null, null];
+
+// Match Python's urllib.parse.quote() behavior
+// https://github.com/python/cpython/blob/3.11/Lib/urllib/parse.py#L826
+// curl and Python let you send non-ASCII characters by encoding each UTF-8 byte.
+// TODO: ignore hex case?
+function _percentEncode(s: string): string {
+  return [...UTF8encoder.encode(s)]
+    .map((b) => {
+      if (
+        // A-Z
+        (b >= 0x41 && b <= 0x5a) ||
+        // a-z
+        (b >= 0x61 && b <= 0x7a) ||
+        // 0-9
+        (b >= 0x30 && b <= 0x39) ||
+        // -._~
+        b === 0x2d ||
+        b === 0x2e ||
+        b === 0x5f ||
+        b === 0x7e
+      ) {
+        return String.fromCharCode(b);
+      }
+      return "%" + b.toString(16).toUpperCase().padStart(2, "0");
+    })
+    .join("");
+}
+
+export function percentEncode(s: Word): Word {
+  const newTokens = [];
+  for (const token of s.tokens) {
+    if (typeof token === "string") {
+      newTokens.push(_percentEncode(token));
+    } else {
+      newTokens.push(token);
+    }
+  }
+  return new Word(newTokens);
+}
+export function percentEncodePlus(s: Word): Word {
+  const newTokens = [];
+  for (const token of s.tokens) {
+    if (typeof token === "string") {
+      newTokens.push(_percentEncode(token).replace(/%20/g, "+"));
+    } else {
+      newTokens.push(token);
+    }
+  }
+  return new Word(newTokens);
+}
+
+// Reimplements decodeURIComponent but ignores variables/commands
+export function wordDecodeURIComponent(s: Word): Word {
+  const newTokens = [];
+  for (const token of s.tokens) {
+    if (typeof token === "string") {
+      newTokens.push(decodeURIComponent(token));
+    } else {
+      newTokens.push(token);
+    }
+  }
+  return new Word(newTokens);
+}
+
+// if url is 'example.com?' the s is ''
+// if url is 'example.com'  the s is null
+export function parseQueryString(s: Word | null): Query {
+  if (!s || s.isEmpty()) {
+    return [null, null];
+  }
+
+  const asList: QueryList = [];
+  for (const param of s.split("&")) {
+    // Most software libraries don't let you distinguish between a=&b= and a&b,
+    // so if we get an `a&b`-type query string, don't bother.
+    if (!param.includes("=")) {
+      return [null, null];
+    }
+
+    const [key, val] = param.split("=", 2);
+    let decodedKey;
+    let decodedVal;
+    try {
+      // https://url.spec.whatwg.org/#urlencoded-parsing
+      // recommends replacing + with space before decoding.
+      decodedKey = wordDecodeURIComponent(key.replace(/\+/g, " "));
+      decodedVal = wordDecodeURIComponent(val.replace(/\+/g, " "));
+    } catch (e) {
+      if (e instanceof URIError) {
+        // Query string contains invalid percent encoded characters,
+        // we cannot properly convert it.
+        return [null, null];
+      }
+      throw e;
+    }
+    // If the query string doesn't round-trip, we cannot properly convert it.
+    // TODO: this is a bit Python-specific, ideally we would check how each runtime/library
+    // percent-encodes query strings. For example, a %27 character in the input query
+    // string will be decoded to a ' but won't be re-encoded into a %27 by encodeURIComponent
+    const roundTripKey = percentEncode(decodedKey);
+    const roundTripVal = percentEncode(decodedVal);
+    // If the original data used %20 instead of + (what requests will send), that's close enough
+    if (
+      (!eq(roundTripKey, key) && !eq(roundTripKey.replace(/%20/g, "+"), key)) ||
+      (!eq(roundTripVal, val) && !eq(roundTripVal.replace(/%20/g, "+"), val))
+    ) {
+      return [null, null];
+    }
+    asList.push([decodedKey, decodedVal]);
+  }
+
+  // Group keys
+  const keyWords: { [key: string]: Word } = {};
+  const uniqueKeys: { [key: string]: Array<Word> } = {};
+  let prevKey = null;
+  for (const [key, val] of asList) {
+    const keyStr = key.toString(); // TODO: do this better
+    if (prevKey === keyStr) {
+      uniqueKeys[keyStr].push(val);
+    } else if (!Object.prototype.hasOwnProperty.call(uniqueKeys, keyStr)) {
+      uniqueKeys[keyStr] = [val];
+      keyWords[keyStr] = key;
+    } else {
+      // If there's a repeated key with a different key between
+      // one of its repetitions, there is no way to represent
+      // this query string as a dictionary.
+      return [asList, null];
+    }
+    prevKey = keyStr;
+  }
+
+  // Convert lists with 1 element to the element
+  const asDict: QueryDict = [];
+  for (const [keyStr, val] of Object.entries(uniqueKeys)) {
+    asDict.push([keyWords[keyStr], val.length === 1 ? val[0] : val]);
+  }
+
+  return [asList, asDict];
+}