@ohos-ports/curlconverter 4.12.0-beta.0

package/src/curl/url.ts

@@ -0,0 +1,241 @@
+import { Word, eq } from "../shell/Word.js";
+import { warnf } from "../Warnings.js";
+import type { GlobalConfig, OperationConfig } from "./opts.js";
+
+// https://github.com/curl/curl/blob/curl-7_87_0/lib/urlapi.c#L60
+// https://github.com/curl/curl/blob/curl-7_87_0/lib/urldata.h#L1295
+export interface Curl_URL {
+  scheme: Word;
+
+  auth?: Word;
+  user?: Word;
+  password?: Word;
+
+  // options: string /* IMAP only? */;
+  host: Word;
+  // zoneid: string /* for numerical IPv6 addresses */;
+  port: Word;
+  path: Word;
+  query: Word;
+  fragment: Word;
+  // portnum: number /* the numerical version */;
+}
+
+// https://github.com/curl/curl/blob/curl-7_88_1/src/tool_urlglob.c#L327
+const MAX_IP6LEN = 128;
+function isIpv6(glob: string): boolean {
+  if (glob.length > MAX_IP6LEN) {
+    return false;
+  }
+  // TODO: curl tries to parse the glob as a hostname.
+  return !glob.includes("-");
+}
+
+function warnAboutGlobs(global_: GlobalConfig, url: string) {
+  // Find any glob expressions in the URL and underline them
+  let prev = "";
+  for (let i = 0; i < url.length; i++) {
+    const cur = url[i];
+    if (cur === "[" && prev !== "\\") {
+      let j = i + 1;
+      while (j < url.length && url[j] !== "]") {
+        j++;
+      }
+      if (j < url.length && url[j] === "]") {
+        const glob = url.slice(i, j + 1);
+        // could be ipv6 address
+        if (!isIpv6(glob)) {
+          warnf(global_, [
+            "glob-in-url",
+            `globs in the URL are not supported:\n` +
+              `${url}\n` +
+              " ".repeat(i) +
+              "^".repeat(glob.length),
+          ]);
+        }
+        prev = "";
+      } else {
+        // No closing bracket
+        warnf(global_, [
+          "unbalanced-glob",
+          "bracket doesn't have a closing bracket:\n" +
+            `${url}\n` +
+            `${" ".repeat(i)}^`,
+        ]);
+        return; // malformed URL, stop looking for globs
+      }
+    } else if (cur === "{" && prev !== "\\") {
+      let j = i + 1;
+      while (j < url.length && url[j] !== "}") {
+        j++;
+      }
+      if (j < url.length && url[j] === "}") {
+        const glob = url.slice(i, j + 1);
+        warnf(global_, [
+          "glob-in-url",
+          `globs in the URL are not supported:\n` +
+            `${url}\n` +
+            " ".repeat(i) +
+            "^".repeat(glob.length),
+        ]);
+        prev = "";
+      } else {
+        // No closing bracket
+        warnf(global_, [
+          "unbalanced-glob",
+          "bracket doesn't have a closing bracket:\n" +
+            `${url}\n` +
+            `${" ".repeat(i)}^`,
+        ]);
+        return; // malformed URL, stop looking for globs
+      }
+    }
+
+    prev = cur;
+  }
+}
+
+export function parseurl(
+  global_: GlobalConfig,
+  config: OperationConfig,
+  url: Word,
+): Curl_URL {
+  // This is curl's parseurl()
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L1144
+  // Except we want to accept all URLs.
+  // curl further validates URLs in curl_url_get()
+  // https://github.com/curl/curl/blob/curl-7_86_0/lib/urlapi.c#L1374
+  const u: Curl_URL = {
+    scheme: new Word(),
+    host: new Word(),
+    port: new Word(),
+    path: new Word(), // with leading '/'
+    query: new Word(), // with leading '?'
+    fragment: new Word(), // with leading '#'
+  };
+
+  // Remove url glob escapes
+  // https://github.com/curl/curl/blob/curl-7_87_0/src/tool_urlglob.c#L395-L398
+  if (!config.globoff) {
+    if (url.isString()) {
+      warnAboutGlobs(global_, url.toString());
+    }
+    url = url.replace(/\\([[\]{}])/g, "$1");
+  }
+
+  // Prepend "http"/"https" if the scheme is missing.
+  // RFC 3986 3.1 says
+  //   scheme      = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+  // but curl will accept a digit/plus/minus/dot in the first character
+  // curl will also accept a url with one / like http:/localhost
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L960
+  let schemeMatch = null;
+  if (url.tokens.length && typeof url.tokens[0] === "string") {
+    schemeMatch = url.tokens[0].match(/^([a-zA-Z0-9+-.]*):\/\/*/);
+  }
+  if (schemeMatch) {
+    const [schemeAndSlashes, scheme] = schemeMatch;
+    u.scheme = new Word(scheme.toLowerCase());
+    url = url.slice(schemeAndSlashes.length);
+  } else {
+    // curl defaults to https://
+    // we don't because most libraries won't downgrade to http
+    // if you ask for https, unlike curl.
+    // TODO: handle file:// scheme
+    u.scheme = config["proto-default"] ?? new Word("http");
+  }
+  if (!eq(u.scheme, "http") && !eq(u.scheme, "https")) {
+    warnf(global_, ["bad-scheme", `Protocol "${u.scheme}" not supported`]);
+  }
+
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L992
+  const hostMatch = url.indexOfFirstChar("/?#");
+  if (hostMatch !== -1) {
+    u.host = url.slice(0, hostMatch);
+    // TODO: u.path might end up empty if indexOfFirstChar found ?#
+    u.path = url.slice(hostMatch); // keep leading '/' in .path
+    // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L1024
+    const fragmentIndex = u.path.indexOf("#");
+    const queryIndex = u.path.indexOf("?");
+    if (fragmentIndex !== -1) {
+      u.fragment = u.path.slice(fragmentIndex);
+      if (queryIndex !== -1 && queryIndex < fragmentIndex) {
+        u.query = u.path.slice(queryIndex, fragmentIndex);
+        u.path = u.path.slice(0, queryIndex);
+      } else {
+        u.path = u.path.slice(0, fragmentIndex);
+      }
+    } else if (queryIndex !== -1) {
+      u.query = u.path.slice(queryIndex);
+      u.path = u.path.slice(0, queryIndex);
+    }
+  } else {
+    u.host = url;
+  }
+
+  // parse username:password@hostname
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L1083
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/urlapi.c#L460
+  // https://github.com/curl/curl/blob/curl-7_85_0/lib/url.c#L2827
+  const authMatch = u.host.indexOf("@");
+  if (authMatch !== -1) {
+    const auth = u.host.slice(0, authMatch);
+    u.host = u.host.slice(authMatch + 1); // throw away '@'
+    if (!config["disallow-username-in-url"]) {
+      u.auth = auth;
+      if (auth.includes(":")) {
+        [u.user, u.password] = auth.split(":", 2);
+      } else {
+        u.user = auth;
+        u.password = new Word(); // if there's no ':', curl will append it
+      }
+    } else {
+      // Curl will exit if this is the case, but we just remove it from the URL
+      warnf(global_, [
+        "login-denied",
+        `Found auth in URL but --disallow-username-in-url was passed: ${auth.toString()}`,
+      ]);
+    }
+  }
+
+  // TODO: need to extract port first
+  // hostname_check()
+  // https://github.com/curl/curl/blob/curl-7_86_0/lib/urlapi.c#L572
+  // if (!u.host) {
+  //   warnf(global_, [
+  //     "no-host",
+  //     "Found empty host in URL: " + JSON.stringify(url),
+  //   ]);
+  // } else if (u.host.startsWith("[")) {
+  //   if (!u.host.endsWith("]")) {
+  //     warnf(global_, [
+  //       "bad-host",
+  //       "Found invalid IPv6 address in URL: " + JSON.stringify(url),
+  //     ]);
+  //   } else {
+  //     const firstWeirdCharacter = u.host.match(/[^0123456789abcdefABCDEF:.]/);
+  //     // %zone_id
+  //     if (firstWeirdCharacter && firstWeirdCharacter[0] !== "%") {
+  //       warnf(global_, [
+  //         "bad-host",
+  //         "Found invalid IPv6 address in URL: " + JSON.stringify(url),
+  //       ]);
+  //     }
+  //   }
+  // } else {
+  //   const firstInvalidCharacter = u.host.match(
+  //     /[\r\n\t/:#?!@{}[\]\\$'"^`*<>=;,]/
+  //   );
+  //   if (firstInvalidCharacter) {
+  //     warnf(global_, [
+  //       "bad-host",
+  //       "Found invalid character " +
+  //         JSON.stringify(firstInvalidCharacter[0]) +
+  //         " in URL: " +
+  //         JSON.stringify(url),
+  //     ]);
+  //   }
+  // }
+
+  return u;
+}

package/src/generators/ansible.ts

@@ -0,0 +1,364 @@
+import { Word, eq } from "../shell/Word.js";
+import { parse, getFirst, COMMON_SUPPORTED_ARGS } from "../parse.js";
+import type { Request, Warnings } from "../parse.js";
+import { parseQueryString, type QueryList, type QueryDict } from "../Query.js";
+
+import yaml from "yamljs";
+
+export const supportedArgs = new Set([
+  ...COMMON_SUPPORTED_ARGS,
+  "compressed",
+  "no-compressed", // only explicitly disabling compression has an effect
+
+  "netrc",
+  "netrc-optional",
+  "no-netrc", // only explicitly disabling netrc has an effect
+  "no-netrc-optional",
+
+  "cacert",
+  "cert",
+  "key",
+  "ciphers",
+  "insecure",
+  "no-insecure",
+
+  "anyauth",
+  "no-anyauth",
+  "digest",
+  "no-digest",
+  "aws-sigv4",
+  "negotiate",
+  "no-negotiate",
+  // "delegation", // GSS/kerberos
+  // Not suported, just more specific warning
+  "ntlm",
+  "no-ntlm",
+  "ntlm-wb",
+  "no-ntlm-wb",
+
+  "unix-socket",
+  "abstract-unix-socket",
+
+  "max-time",
+
+  "location",
+  "no-location",
+  "location-trusted",
+  "no-location-trusted",
+
+  "upload-file",
+  "output",
+
+  "form",
+  "form-string",
+]);
+
+// https://docs.ansible.com/ansible/latest/collections/ansible/builtin/uri_module.html
+type BodyFormat =
+  | "form-urlencoded"
+  | "json"
+  | "raw" // default
+  | "form-multipart";
+type AnsibleForm = {
+  [key: string]: {
+    content?: string;
+    filename?: string;
+    mime_type?: string;
+  };
+};
+type Body = QueryList | QueryDict | object | string | AnsibleForm;
+type AnsibleURI = {
+  url: string;
+  method?: string;
+  headers?: { [key: string]: string };
+  body?: Body;
+  body_format?: BodyFormat;
+  src?: string; // shouldn't have body if has src
+  dest?: string;
+
+  url_username?: string;
+  url_password?: string;
+  force_basic_auth?: boolean;
+  use_gssapi?: boolean;
+  use_netrc?: boolean; // true
+
+  // String won't work, it's in case we can't parse it as a number
+  timeout?: number | string; // 30
+
+  decompress?: boolean;
+  follow_redirects?:
+    | "all"
+    | "no"
+    | "none"
+    | "safe" // default
+    | "urllib2"
+    | "yes";
+
+  ca_path?: string;
+  ciphers?: string;
+  client_cert?: string;
+  client_key?: string;
+  validate_certs?: boolean;
+
+  unix_socket?: string;
+};
+
+function getDataString(
+  request: Request,
+  warnings: Warnings,
+): [Body, BodyFormat] | [string, "src"] | undefined {
+  if (!request.data || !request.data.isString()) {
+    return;
+  }
+  const data = request.data.toString();
+
+  const contentType = request.headers.getContentType();
+  // TODO: delete Content-Type header when it's what Ansible will send anyway
+  // const exactContentType = request.headers.get("Content-Type");
+  if (contentType === "application/json") {
+    try {
+      const dataAsJson = JSON.parse(data);
+      // TODO: if doesn't roundtrip, add commented out raw string
+      // TODO: we actually want to know how it's serialized by Ansible
+      // const roundtrips = JSON.stringify(dataAsJson) === dataStr;
+      return [dataAsJson, "json"];
+    } catch {}
+  } else if (contentType === "application/x-www-form-urlencoded") {
+    if (
+      request.dataArray &&
+      request.dataArray.length === 1 &&
+      !(request.dataArray[0] instanceof Word) &&
+      !request.dataArray[0].name
+    ) {
+      const filetype = request.dataArray[0].filetype;
+      const filename = request.dataArray[0].filename.toString();
+      // TODO: and newlines have to be stripped for others
+      if (filetype === "urlencode") {
+        // TODO: or the other ones don't need to be?
+        warnings.push([
+          "urlencoded-file",
+          "the file needs to be urlencoded: " + JSON.stringify(filename),
+        ]);
+      }
+      return [filename, "src"];
+    }
+    const [queryList, queryDict] = parseQueryString(request.data);
+    if (queryDict) {
+      return [
+        Object.fromEntries(
+          queryDict.map((q) => [
+            q[0].toString(),
+            Array.isArray(q[1])
+              ? q[1].map((qv) => qv.toString())
+              : q[1].toString(),
+          ]),
+        ),
+        "form-urlencoded",
+      ];
+    }
+    if (queryList) {
+      return [
+        queryList.map((q) => [q[0].toString(), q[1].toString()]),
+        "form-urlencoded",
+      ];
+    }
+  }
+  return;
+}
+
+export function _toAnsible(
+  requests: Request[],
+  warnings: Warnings = [],
+): string {
+  // Only supported if there's one file and nothing else
+  const request = getFirst(requests, warnings, { dataReadsFile: true });
+  if (
+    request.dataReadsFile &&
+    request.dataArray &&
+    (request.dataArray.length > 1 ||
+      (!(request.dataArray[0] instanceof Word) && request.dataArray[0].name))
+  ) {
+    warnings.push([
+      "unsafe-data",
+      "the generated body is wrong, " +
+        JSON.stringify("@" + request.dataReadsFile) +
+        " means read the file " +
+        JSON.stringify(request.dataReadsFile),
+    ]);
+  }
+
+  const r: AnsibleURI = {
+    url: request.urls[0].url.toString(),
+    method: request.urls[0].method.toString(),
+  };
+  if (request.multipartUploads) {
+    const form: AnsibleForm = {};
+    for (const m of request.multipartUploads) {
+      // TODO: can't have duplicate keys
+      const name = m.name.toString();
+      form[name] = {};
+      if ("content" in m) {
+        form[name].content = m.content.toString();
+      } else {
+        // TODO: get basename
+        form[name].filename = m.contentFile.toString();
+        if (m.filename && !eq(m.filename, m.contentFile)) {
+          warnings.push([
+            "multipart-fake-filename",
+            // TODO: check if this is true, the only source is it's not in the example
+            // https://docs.ansible.com/ansible/latest/collections/ansible/builtin/uri_module.html#examples
+            "Ansible doesn't allow reading from a file and changing the sent filename: " +
+              JSON.stringify(m.contentFile.toString()),
+          ]);
+        }
+      }
+      if ("contentType" in m && m.contentType) {
+        form[name].mime_type = m.contentType.toString();
+      }
+    }
+    r.body = form;
+    r.body_format = "form-multipart";
+  } else if (request.data) {
+    const d = getDataString(request, warnings);
+    if (d) {
+      const [body, format] = d;
+      if (format === "src") {
+        r.src = body;
+      } else {
+        r.body = body;
+        if (format !== "raw") {
+          r.body_format = format;
+        }
+      }
+    } else {
+      r.body = request.data.toString();
+    }
+  } else if (request.urls[0].uploadFile) {
+    r.src = request.urls[0].uploadFile.toString();
+  }
+  if (request.urls[0].output) {
+    r.dest = request.urls[0].output.toString();
+  }
+
+  if (request.headers.length) {
+    r.headers = {};
+    for (const h of request.headers) {
+      const [k, v] = h;
+      if (v !== null) {
+        r.headers[k.toString()] = v.toString();
+      }
+    }
+  }
+  if (request.urls[0].auth) {
+    const [username, password] = request.urls[0].auth;
+    if (username.toBool()) {
+      r.url_username = username.toString();
+    }
+    if (password.toBool()) {
+      r.url_password = password.toString();
+    }
+  }
+  // No idea if this stuff is correct
+  switch (request.authType) {
+    case "basic":
+      // TODO: if --basic explicitly passed, set this?
+      // r.force_basic_auth = true;
+      break;
+    case "negotiate":
+      r.use_gssapi = true;
+      break;
+    case "ntlm":
+    case "ntlm-wb":
+      // https://docs.ansible.com/ansible/latest/collections/ansible/builtin/uri_module.html#parameter-use_gssapi
+      warnings.push(["ntlm", "Ansible doesn't support NTLM authentication."]);
+      break;
+  }
+
+  if (request.cacert) {
+    r.ca_path = request.cacert.toString();
+  }
+  if (request.cert) {
+    // TODO: might have password
+    const [cert, password] = request.cert;
+    r.client_cert = cert.toString();
+    if (password) {
+      warnings.push([
+        "cert-password",
+        "Ansible does not support client certificate passwords: " +
+          JSON.stringify(password.toString()),
+      ]);
+    }
+  }
+  if (request.key) {
+    r.client_key = request.key.toString();
+  }
+  if (request.ciphers) {
+    r.ciphers = request.ciphers.toString();
+  }
+  if (request.insecure) {
+    r.validate_certs = false;
+  }
+  if (request.netrc === "ignored") {
+    r.use_netrc = false;
+  }
+
+  if (request.compressed === false) {
+    r.decompress = false;
+  }
+  // curl defaults to not following redirects, Ansible defaults to following
+  // safe redirects. Don't change that default unless it's explicitly set.
+  if (request.followRedirects === false) {
+    r.follow_redirects = "none";
+  } else if (request.followRedirects === true) {
+    // curl will follow redirects to a different host but not send credentials
+    // Ansible will only follow redirects for GET or HEAD to the same host
+    r.follow_redirects = request.followRedirectsTrusted ? "all" : "safe";
+  }
+
+  if (request.timeout) {
+    const timeoutStr = request.timeout.toString();
+    const timeoutInt = parseInt(timeoutStr); // TODO: warn if wasn't int
+
+    if (isNaN(timeoutInt)) {
+      r.timeout = timeoutStr;
+      warnings.push([
+        "ansible-timeout-not-int",
+        "couldn't parse timeout as an integer: " + JSON.stringify(timeoutStr),
+      ]);
+    } else {
+      r.timeout = timeoutInt;
+    }
+
+    warnings.push([
+      "ansible-timeout",
+      "Ansible's timeout is socket-level but curl's is for the entire request.",
+    ]);
+  }
+
+  if (request.unixSocket) {
+    r.unix_socket = request.unixSocket.toString();
+  }
+
+  return yaml.stringify(
+    [
+      {
+        name: request.urls[0].urlWithoutQueryList.toString(),
+        uri: r,
+        register: "result",
+      },
+    ],
+    100,
+    2,
+  );
+}
+export function toAnsibleWarn(
+  curlCommand: string | string[],
+  warnings: Warnings = [],
+): [string, Warnings] {
+  const requests = parse(curlCommand, supportedArgs, warnings);
+  const ansible = _toAnsible(requests, warnings);
+  return [ansible, warnings];
+}
+export function toAnsible(curlCommand: string | string[]): string {
+  return toAnsibleWarn(curlCommand)[0];
+}