@ohos-ports/curlconverter 4.12.0-beta.0

package/src/generators/wget.ts

@@ -0,0 +1,533 @@
+import { warnIfPartsIgnored } from "../Warnings.js";
+import { Word, eq, mergeWords } from "../shell/Word.js";
+import { parse, COMMON_SUPPORTED_ARGS } from "../parse.js";
+import type { Request, Warnings } from "../parse.js";
+
+export const supportedArgs = new Set([
+  ...COMMON_SUPPORTED_ARGS,
+  "cookie-jar",
+
+  "ipv4",
+  "no-ipv4",
+  "ipv6",
+  "no-ipv6",
+
+  "form", // not supported, more specific warning
+  "form-string",
+
+  "ciphers",
+  "insecure",
+  "cert",
+  "cert-type",
+  "key",
+  "key-type",
+  "cacert",
+  "capath",
+  "crlfile",
+  "pinnedpubkey",
+  "random-file",
+  "egd-file",
+  "hsts",
+
+  "noproxy", // not fully supported
+  "proxy", // not supported
+  "proxy-user",
+
+  // TODO: supported by all?
+  "globoff",
+  "no-globoff",
+
+  "netrc",
+  "netrc-optional",
+
+  "timeout",
+  "connect-timeout",
+  "limit-rate",
+
+  "compressed",
+  "no-compressed",
+
+  // Wget picks the auth and some it doesn't support but there's a more
+  // specific error message for those.
+  "anyauth",
+  "no-anyauth",
+  "digest",
+  "no-digest",
+  "aws-sigv4",
+  "negotiate",
+  "no-negotiate",
+  "delegation", // GSS/kerberos
+  // "service-name", // GSS/kerberos, not supported
+  "ntlm",
+  "no-ntlm",
+  "ntlm-wb",
+  "no-ntlm-wb",
+  "no-digest",
+
+  "output",
+  "upload-file",
+
+  "continue-at",
+  "clobber",
+  "remote-time",
+
+  "keep-alive",
+
+  "silent",
+  "verbose",
+
+  "next",
+]);
+
+// TODO: check this
+// https://www.gnu.org/software/bash/manual/html_node/Quoting.html
+const regexDoubleEscape = /\$|`|"|\\|!/gu;
+const unprintableChars = /\p{C}|[^ \P{Z}]/gu; // TODO: there's probably more
+const regexAnsiCEscape = /\p{C}|[^ \P{Z}]|\\|'/gu;
+// https://unix.stackexchange.com/questions/270977/
+const shellChars = /[\x02-\x09\x0b-\x1a\\#?`(){}[\]^*<=>~|; "!$&'\x82-\xff]/;
+export function reprStr(s: string, mustQuote = false): string {
+  const containsUnprintableChars = unprintableChars.test(s);
+  if (containsUnprintableChars) {
+    return (
+      "$'" +
+      s.replace(regexAnsiCEscape, (c: string) => {
+        switch (c) {
+          case "\x07":
+            return "\\a";
+          case "\b":
+            return "\\b";
+          case "\x1B":
+            return "\\e";
+          case "\f":
+            return "\\f";
+          case "\n":
+            return "\\n";
+          case "\r":
+            return "\\r";
+          case "\t":
+            return "\\t";
+          case "\v":
+            return "\\v";
+          case "\\":
+            return "\\\\";
+          case "'":
+            return "\\'";
+          // TODO: unnecessary?
+          //   case '"':
+          //     return '\\"';
+          //   case "?":
+          //     return "\\?";
+        }
+
+        const hex = (c.codePointAt(0) as number).toString(16);
+        if (hex.length <= 2) {
+          return "\\x" + hex.padStart(2, "0");
+        }
+        if (hex.length <= 4) {
+          return "\\u" + hex.padStart(4, "0");
+        }
+        return "\\U" + hex.padStart(8, "0");
+      }) +
+      "'"
+    );
+  }
+  if (
+    (s.includes('"') ||
+      s.includes("$") ||
+      s.includes("`") ||
+      s.includes("!") ||
+      mustQuote) &&
+    !s.includes("'")
+  ) {
+    return "'" + s + "'";
+  }
+  if (shellChars.test(s) || mustQuote) {
+    return (
+      '"' +
+      s.replace(regexDoubleEscape, (c: string) => {
+        switch (c[0]) {
+          case "$":
+            return "\\$";
+          case "`":
+            return "\\`";
+          case '"':
+            return '\\"';
+          case "\\":
+            return "\\\\";
+          case "!":
+            // ! might have an effect or it might not, depending on if
+            // the command is in a file or pasted into the shell and other things.
+            // If it's backslash escaped, the backslash is still not removed, a
+            // safe way to escape it is to close the double quote, put the ! in
+            // single quotes and then open the double quote again.
+            return "\"'!'\"";
+        }
+        // Should never happen
+        return c;
+      }) +
+      '"'
+    );
+  }
+  return s;
+}
+
+export function repr(w: Word): string {
+  // TODO: put variables in the quotes
+  const args: string[] = [];
+  for (const [i, t] of w.tokens.entries()) {
+    if (typeof t === "string") {
+      // A string after a variable cannot be un-quoted
+      // After a subcommand it's usually okay but that's more complex.
+      args.push(reprStr(t, !!i));
+    } else if (t.type === "variable") {
+      args.push(t.text);
+    } else {
+      // TODO: this will probably lead to syntax errors
+      args.push(t.text);
+    }
+  }
+  return args.join("");
+}
+
+function requestToWget(request: Request, warnings: Warnings): string {
+  warnIfPartsIgnored(request, warnings, {
+    dataReadsFile: true,
+    cookieFiles: true,
+    multipleUrls: true,
+  });
+  if (
+    request.dataReadsFile &&
+    request.dataArray &&
+    request.dataArray.length &&
+    (request.dataArray.length > 1 ||
+      (!(request.dataArray[0] instanceof Word) && request.dataArray[0].name))
+  ) {
+    warnings.push([
+      "unsafe-data",
+      "the generated data content is wrong, " +
+        // TODO: might not come from "@"
+        JSON.stringify("@" + request.dataReadsFile) +
+        " means read the file " +
+        JSON.stringify(request.dataReadsFile),
+    ]);
+  }
+
+  const args: string[] = [];
+
+  if (
+    request.urls[0].uploadFile ||
+    (request.dataArray &&
+      request.dataArray.length === 1 &&
+      !(request.dataArray[0] instanceof Word) &&
+      !request.dataArray[0].name) ||
+    request.data
+  ) {
+    if (!eq(request.urls[0].method, "POST")) {
+      args.push("--method=" + repr(request.urls[0].method));
+    }
+  } else if (!eq(request.urls[0].method, "GET")) {
+    args.push("--method=" + repr(request.urls[0].method));
+  }
+  if (request.urls.length > 1) {
+    const uniqueMethods = new Set<string>(
+      request.urls.map((u) => u.method.toString()),
+    );
+
+    // TODO: add tons of checks/warnings that wget doesn't let you set things per-URL
+    if (uniqueMethods.size > 1) {
+      warnings.push([
+        "mixed-methods",
+        "the input curl command uses multiple HTTP methods, which Wget doesn't support: " +
+          request.urls
+            .map((u) => JSON.stringify(u.method.toString()))
+            .join(", "),
+      ]);
+    }
+  }
+
+  if (request.cookieFiles && request.cookieFiles.length) {
+    for (const cookieFile of request.cookieFiles) {
+      args.push("--load-cookies=" + repr(cookieFile));
+    }
+
+    if (request.cookieFiles.length > 1) {
+      const lastCookieFile =
+        request.cookieFiles[request.cookieFiles.length - 1];
+      warnings.push([
+        "multiple-cookie-files",
+        "Wget only supports reading cookies from one file, only the last one will be sent: " +
+          lastCookieFile.toString(),
+      ]);
+    }
+  }
+  if (request.cookieJar) {
+    args.push("--save-cookies=" + repr(request.cookieJar));
+    // TODO: --keep-session-cookies
+  }
+
+  if (request.headers.length) {
+    for (const [headerName, headerValue] of request.headers) {
+      args.push(
+        "--header=" + repr(mergeWords(headerName, ": ", headerValue ?? "")),
+      );
+      // TODO: there's also --referer, --user-agent and --content-disposition
+    }
+  }
+
+  if (request.compressed !== undefined) {
+    if (request.compressed) {
+      args.push("--compression=auto");
+    } else {
+      args.push("--compression=none");
+    }
+  }
+
+  if (request.maxRedirects && request.maxRedirects.toString() !== "20") {
+    // TODO: escape/parse?
+    args.push("--max-redirect=" + repr(request.maxRedirects));
+  }
+
+  if (request.ipv4) {
+    args.push("--inet4-only");
+  }
+  if (request.ipv6) {
+    args.push("--inet6-only");
+  }
+
+  if (request.urls[0].auth) {
+    const [user, password] = request.urls[0].auth;
+    args.push("--user=" + repr(user));
+    if (password.toBool()) {
+      args.push("--password=" + repr(password));
+    }
+    if (request.authType === "basic") {
+      args.push("--auth-no-challenge");
+    }
+
+    if (
+      !["none", "basic", "digest", "ntlm", "ntlm-wb", "negotiate"].includes(
+        request.authType,
+      )
+    ) {
+      warnings.push([
+        "wget-unsupported-auth",
+        "Wget doesn't support " + request.authType + " authentication",
+      ]);
+    }
+  }
+
+  if (request.urls[0].uploadFile) {
+    if (
+      eq(request.urls[0].uploadFile, "-") ||
+      eq(request.urls[0].uploadFile, ".")
+    ) {
+      warnings.push([
+        "wget-stdin",
+        "Wget does not support uploading from stdin",
+      ]);
+    }
+    if (eq(request.urls[0].method, "POST")) {
+      args.push("--post-file=" + repr(request.urls[0].uploadFile));
+    } else {
+      args.push("--body-file=" + repr(request.urls[0].uploadFile));
+    }
+  } else if (request.multipartUploads) {
+    warnings.push([
+      "multipart",
+      "Wget does not support sending multipart/form-data",
+    ]);
+  } else if (
+    request.dataArray &&
+    request.dataArray.length === 1 &&
+    !(request.dataArray[0] instanceof Word) &&
+    !request.dataArray[0].name
+  ) {
+    if (eq(request.urls[0].method, "POST")) {
+      args.push("--post-file=" + repr(request.dataArray[0].filename));
+    } else {
+      args.push("--body-file=" + repr(request.dataArray[0].filename));
+    }
+  } else if (request.data) {
+    if (eq(request.urls[0].method, "POST")) {
+      args.push("--post-data=" + repr(request.data));
+    } else {
+      args.push("--body-data=" + repr(request.data));
+    }
+  }
+
+  // https://www.gnu.org/software/wget/manual/html_node/HTTPS-_0028SSL_002fTLS_0029-Options.html
+  if (request.ciphers) {
+    args.push("--ciphers=" + repr(request.ciphers));
+  }
+  if (request.insecure) {
+    args.push("--no-check-certificate");
+  }
+  if (request.cert) {
+    const [cert, password] = request.cert;
+    args.push("--certificate=" + repr(cert));
+    if (password) {
+      warnings.push([
+        "wget-cert-password",
+        // TODO: is this true?
+        "Wget does not support certificate passwords",
+      ]);
+    }
+  }
+  if (request.certType) {
+    args.push("--certificate-type=" + repr(request.certType));
+  }
+  if (request.key) {
+    args.push("--private-key=" + repr(request.key));
+  }
+  if (request.keyType) {
+    args.push("--private-key-type=" + repr(request.keyType));
+  }
+
+  if (request.cacert) {
+    args.push("--ca-certificate=" + repr(request.cacert));
+  }
+  if (request.capath) {
+    args.push("--ca-directory=" + repr(request.capath));
+  }
+  if (request.crlfile) {
+    args.push("--crl-file=" + repr(request.crlfile));
+  }
+  if (request.pinnedpubkey) {
+    args.push("--pinnedpubkey=" + repr(request.pinnedpubkey));
+  }
+  if (request.randomFile) {
+    args.push("--random-file=" + repr(request.randomFile));
+  }
+  if (request.egdFile) {
+    args.push("--egd-file=" + repr(request.egdFile));
+  }
+  if (request.hsts) {
+    for (const hsts of request.hsts) {
+      // TODO: warn multiple won't work
+      args.push("--hsts-file=" + repr(hsts));
+    }
+  }
+  // TODO: --secure-protocol
+
+  if (request.netrc === "ignored") {
+    args.push("--no-netrc");
+  }
+
+  if (request.noproxy) {
+    if (eq(request.noproxy, "*")) {
+      args.push("--no-proxy");
+    } else {
+      warnings.push([
+        "wget-noproxy",
+        "Wget does not support noproxy for specific hosts",
+      ]);
+    }
+  }
+  if (request.proxy) {
+    warnings.push([
+      "wget-proxy",
+      "Wget requires specifying proxies in environment variables: " +
+        JSON.stringify(request.proxy.toString()),
+    ]);
+  }
+  if (request.proxyAuth) {
+    const [proxyUser, proxyPassword] = request.proxyAuth.split(":", 2);
+    args.push("--proxy-user=" + repr(proxyUser));
+    if (proxyPassword && proxyPassword.toBool()) {
+      args.push("--proxy-password=" + repr(proxyPassword));
+    }
+  }
+
+  if (request.timeout) {
+    // TODO: warn that this is not for the whole request
+    args.push("--timeout=" + repr(request.timeout));
+  }
+  if (request.connectTimeout) {
+    args.push("--connect-timeout=" + repr(request.connectTimeout));
+  }
+
+  // TODO: curl's --speed-limit
+  if (request.limitRate) {
+    // TODO: they probably have different syntaxes
+    args.push("--limit-rate=" + repr(request.limitRate));
+  }
+
+  // TODO: this doesn't match up. Warn if multiple, etc.
+  if (request.urls[0].output) {
+    args.push("--output-document=" + repr(request.urls[0].output));
+  } else {
+    args.push("--output-document -");
+  }
+  if (request.clobber === false) {
+    args.push("--no-clobber");
+  }
+  // TODO: --create-dirs
+
+  if (request.remoteTime === false) {
+    args.push("--no-use-server-timestamps");
+  }
+
+  if (request.continueAt) {
+    if (eq(request.continueAt, "-")) {
+      args.push("--continue");
+    } else {
+      args.push("--start-pos=" + repr(request.continueAt));
+    }
+  }
+
+  if (request.keepAlive === false) {
+    args.push("--no-http-keep-alive");
+  }
+
+  // This is used only for FTP by Wget, we might as well add it even
+  // though we don't support FTP.
+  if (request.globoff) {
+    // TODO: this isn't great, it has different meanings from curl to Wget
+    // TODO: don't remove \[ and \] from original URL?
+    args.push("--no-glob");
+  }
+
+  // TODO:
+  // curl:
+  // --silent, --verbose, --no-verbose (the default)
+  // --progress-meter (the default), --no-progress-meter, --progress-bar
+  // --trace-ascii, --trace
+  // wget:
+  // --quiet, --verbose (the default), --no-verbose
+  // --progress=bar (the default), --progress=dot
+  // --debug
+  if (request.silent) {
+    args.push("--quiet");
+  } else if (request.verbose === false) {
+    args.push("--no-verbose");
+  }
+
+  for (const url of request.urls) {
+    // TODO: wget probably has a different syntax
+    args.push(repr(url.url));
+  }
+
+  const multiline =
+    args.length > 3 || args.reduce((a, b) => a + b.length, 0) > 80 - 5;
+  const joiner = multiline ? " \\\n  " : " ";
+  return "wget " + args.join(joiner) + "\n";
+}
+
+// TODO: --verbose ?
+export function _toWget(requests: Request[], warnings: Warnings = []): string {
+  return requests.map((r) => requestToWget(r, warnings)).join("\n\n");
+}
+
+export function toWgetWarn(
+  curlCommand: string | string[],
+  warnings: Warnings = [],
+): [string, Warnings] {
+  const requests = parse(curlCommand, supportedArgs, warnings);
+  const wget = _toWget(requests, warnings);
+  return [wget, warnings];
+}
+
+export function toWget(curlCommand: string | string[]): string {
+  return toWgetWarn(curlCommand)[0];
+}

package/src/index.ts

@@ -0,0 +1,89 @@
+export { toAnsible, toAnsibleWarn } from "./generators/ansible.js";
+export { toC, toCWarn } from "./generators/c.js";
+export { toCFML, toCFMLWarn } from "./generators/cfml.js";
+export { toClojure, toClojureWarn } from "./generators/clojure.js";
+export { toCSharp, toCSharpWarn } from "./generators/csharp.js";
+export { toDart, toDartWarn } from "./generators/dart.js";
+export { toElixir, toElixirWarn } from "./generators/elixir.js";
+export { toGo, toGoWarn } from "./generators/go.js";
+export { toHarString, toHarStringWarn } from "./generators/har.js";
+export { toHTTP, toHTTPWarn } from "./generators/http.js";
+export { toHttpie, toHttpieWarn } from "./generators/httpie.js";
+export { toJava, toJavaWarn } from "./generators/java/java.js";
+export {
+  toJavaHttpUrlConnection,
+  toJavaHttpUrlConnectionWarn,
+} from "./generators/java/httpurlconnection.js";
+export { toJavaJsoup, toJavaJsoupWarn } from "./generators/java/jsoup.js";
+export { toJavaOkHttp, toJavaOkHttpWarn } from "./generators/java/okhttp.js";
+export {
+  toJavaScript,
+  toJavaScriptWarn,
+} from "./generators/javascript/javascript.js";
+export {
+  toJavaScriptJquery,
+  toJavaScriptJqueryWarn,
+} from "./generators/javascript/jquery.js";
+export {
+  toJavaScriptXHR,
+  toJavaScriptXHRWarn,
+} from "./generators/javascript/xhr.js";
+export {
+  toJsonObject,
+  toJsonObjectWarn,
+  toJsonString,
+  toJsonStringWarn,
+  type JSONOutput,
+} from "./generators/json.js";
+export { toJulia, toJuliaWarn } from "./generators/julia.js";
+export { toKotlin, toKotlinWarn } from "./generators/kotlin.js";
+export { toLua, toLuaWarn } from "./generators/lua.js";
+export { toMATLAB, toMATLABWarn } from "./generators/matlab/matlab.js";
+export { toNode, toNodeWarn } from "./generators/javascript/javascript.js";
+export { toNodeAxios, toNodeAxiosWarn } from "./generators/javascript/axios.js";
+export { toNodeGot, toNodeGotWarn } from "./generators/javascript/got.js";
+export { toNodeHttp, toNodeHttpWarn } from "./generators/javascript/http.js";
+export { toNodeKy, toNodeKyWarn } from "./generators/javascript/ky.js";
+export {
+  toNodeSuperAgent,
+  toNodeSuperAgentWarn,
+} from "./generators/javascript/superagent.js";
+export {
+  toNodeRequest,
+  toNodeRequestWarn,
+} from "./generators/javascript/request.js";
+export { toObjectiveC, toObjectiveCWarn } from "./generators/objectivec.js";
+export { toOCaml, toOCamlWarn } from "./generators/ocaml.js";
+export { toPerl, toPerlWarn } from "./generators/perl.js";
+export { toPhp, toPhpWarn } from "./generators/php/php.js";
+export { toPhpGuzzle, toPhpGuzzleWarn } from "./generators/php/guzzle.js";
+export { toPhpRequests, toPhpRequestsWarn } from "./generators/php/requests.js";
+export {
+  toPowershellRestMethod,
+  toPowershellRestMethodWarn,
+} from "./generators/powershell.js";
+export {
+  toPowershellWebRequest,
+  toPowershellWebRequestWarn,
+} from "./generators/powershell.js";
+export { toPython, toPythonWarn } from "./generators/python/python.js";
+export { toPythonHttp, toPythonHttpWarn } from "./generators/python/http.js";
+export { toR, toRWarn } from "./generators/r/httr.js";
+export { toRHttr2, toRHttr2Warn } from "./generators/r/httr2.js";
+export { toRuby, toRubyWarn } from "./generators/ruby/ruby.js";
+export {
+  toRubyHttparty,
+  toRubyHttpartyWarn,
+} from "./generators/ruby/httparty.js";
+export { toRust, toRustWarn } from "./generators/rust.js";
+export { toSwift, toSwiftWarn } from "./generators/swift.js";
+export { toWget, toWgetWarn } from "./generators/wget.js";
+
+// backwards compatibility aliases
+export { toJavaScript as toBrowser } from "./generators/javascript/javascript.js";
+export { toNode as toNodeFetch } from "./generators/javascript/javascript.js";
+export { toJavaScriptWarn as toBrowserWarn } from "./generators/javascript/javascript.js";
+export { toNodeWarn as toNodeFetchWarn } from "./generators/javascript/javascript.js";
+
+export { CCError } from "./utils.js";
+export type { Warnings } from "./Warnings.js";

package/src/parse.ts

@@ -0,0 +1,72 @@
+import { CCError } from "./utils.js";
+import { Word } from "./shell/Word.js";
+import { tokenize } from "./shell/tokenizer.js";
+import * as curl from "./curl/opts.js";
+import {
+  curlLongOpts,
+  curlLongOptsShortened,
+  curlShortOpts,
+} from "./curl/opts.js";
+import { buildRequests } from "./Request.js";
+import type { Request, RequestUrl } from "./Request.js";
+import type { Warnings } from "./Warnings.js";
+
+export { COMMON_SUPPORTED_ARGS } from "./curl/opts.js";
+export { getFirst } from "./Request.js";
+
+export type { Request, RequestUrl, Warnings };
+
+export function clip(s: string, maxLength = 30): string {
+  if (s.length > maxLength) {
+    return s.slice(0, maxLength - 3) + "...";
+  }
+  return s;
+}
+
+function findCommands(
+  curlCommand: string | string[],
+  warnings: Warnings,
+): [Word[], Word?, Word?][] {
+  if (typeof curlCommand === "string") {
+    return tokenize(curlCommand, warnings);
+  }
+
+  if (curlCommand.length === 0) {
+    throw new CCError("no arguments provided");
+  }
+  if (curlCommand[0].trim() !== "curl") {
+    throw new CCError(
+      'command should begin with "curl" but instead begins with ' +
+        JSON.stringify(clip(curlCommand[0])),
+    );
+  }
+  return [[curlCommand.map((arg) => new Word(arg)), undefined, undefined]];
+}
+
+/**
+ * Accepts a string of Bash code or a tokenized argv array.
+ * Returns an array of parsed curl objects.
+ * @param command a string of Bash code containing at least one curl command or an
+ * array of shell argument tokens (meant for passing process.argv).
+ */
+export function parse(
+  command: string | string[],
+  supportedArgs?: Set<string>,
+  warnings: Warnings = [],
+): Request[] {
+  let requests: Request[] = [];
+  const curlCommands = findCommands(command, warnings);
+  for (const [argv, stdin, stdinFile] of curlCommands) {
+    const [globalConfig] = curl.parseArgs(
+      argv,
+      curlLongOpts,
+      curlLongOptsShortened,
+      curlShortOpts,
+      supportedArgs,
+      warnings,
+    );
+
+    requests = requests.concat(buildRequests(globalConfig, stdin, stdinFile));
+  }
+  return requests;
+}

package/src/shell/Parser.ts

@@ -0,0 +1,8 @@
+import Parser from "@ohos-ports/tree-sitter";
+import Bash from "@ohos-ports/tree-sitter-bash";
+
+const parser = new Parser();
+parser.setLanguage(Bash);
+
+export default parser;
+export type { Parser };