@ohos-ports/curlconverter 4.12.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +153 -0
- package/dist/src/Headers.d.ts +23 -0
- package/dist/src/Headers.js +287 -0
- package/dist/src/Headers.js.map +1 -0
- package/dist/src/Query.d.ts +8 -0
- package/dist/src/Query.js +136 -0
- package/dist/src/Query.js.map +1 -0
- package/dist/src/Request.d.ts +200 -0
- package/dist/src/Request.js +1191 -0
- package/dist/src/Request.js.map +1 -0
- package/dist/src/Warnings.d.ts +15 -0
- package/dist/src/Warnings.js +111 -0
- package/dist/src/Warnings.js.map +1 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +333 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/curl/auth.d.ts +12 -0
- package/dist/src/curl/auth.js +40 -0
- package/dist/src/curl/auth.js.map +1 -0
- package/dist/src/curl/form.d.ts +28 -0
- package/dist/src/curl/form.js +218 -0
- package/dist/src/curl/form.js.map +1 -0
- package/dist/src/curl/opts.d.ts +2052 -0
- package/dist/src/curl/opts.js +1000 -0
- package/dist/src/curl/opts.js.map +1 -0
- package/dist/src/curl/url.d.ts +14 -0
- package/dist/src/curl/url.js +218 -0
- package/dist/src/curl/url.js.map +1 -0
- package/dist/src/generators/ansible.d.ts +5 -0
- package/dist/src/generators/ansible.js +282 -0
- package/dist/src/generators/ansible.js.map +1 -0
- package/dist/src/generators/c.d.ts +12 -0
- package/dist/src/generators/c.js +1230 -0
- package/dist/src/generators/c.js.map +1 -0
- package/dist/src/generators/cfml.d.ts +5 -0
- package/dist/src/generators/cfml.js +134 -0
- package/dist/src/generators/cfml.js.map +1 -0
- package/dist/src/generators/clojure.d.ts +7 -0
- package/dist/src/generators/clojure.js +436 -0
- package/dist/src/generators/clojure.js.map +1 -0
- package/dist/src/generators/csharp.d.ts +8 -0
- package/dist/src/generators/csharp.js +356 -0
- package/dist/src/generators/csharp.js.map +1 -0
- package/dist/src/generators/dart.d.ts +5 -0
- package/dist/src/generators/dart.js +273 -0
- package/dist/src/generators/dart.js.map +1 -0
- package/dist/src/generators/elixir.d.ts +8 -0
- package/dist/src/generators/elixir.js +296 -0
- package/dist/src/generators/elixir.js.map +1 -0
- package/dist/src/generators/go.d.ts +5 -0
- package/dist/src/generators/go.js +228 -0
- package/dist/src/generators/go.js.map +1 -0
- package/dist/src/generators/har.d.ts +7 -0
- package/dist/src/generators/har.js +115 -0
- package/dist/src/generators/har.js.map +1 -0
- package/dist/src/generators/http.d.ts +7 -0
- package/dist/src/generators/http.js +165 -0
- package/dist/src/generators/http.js.map +1 -0
- package/dist/src/generators/httpie.d.ts +5 -0
- package/dist/src/generators/httpie.js +446 -0
- package/dist/src/generators/httpie.js.map +1 -0
- package/dist/src/generators/java/httpurlconnection.d.ts +5 -0
- package/dist/src/generators/java/httpurlconnection.js +123 -0
- package/dist/src/generators/java/httpurlconnection.js.map +1 -0
- package/dist/src/generators/java/java.d.ts +8 -0
- package/dist/src/generators/java/java.js +247 -0
- package/dist/src/generators/java/java.js.map +1 -0
- package/dist/src/generators/java/jsoup.d.ts +5 -0
- package/dist/src/generators/java/jsoup.js +152 -0
- package/dist/src/generators/java/jsoup.js.map +1 -0
- package/dist/src/generators/java/okhttp.d.ts +5 -0
- package/dist/src/generators/java/okhttp.js +232 -0
- package/dist/src/generators/java/okhttp.js.map +1 -0
- package/dist/src/generators/javascript/axios.d.ts +5 -0
- package/dist/src/generators/javascript/axios.js +332 -0
- package/dist/src/generators/javascript/axios.js.map +1 -0
- package/dist/src/generators/javascript/got.d.ts +5 -0
- package/dist/src/generators/javascript/got.js +277 -0
- package/dist/src/generators/javascript/got.js.map +1 -0
- package/dist/src/generators/javascript/http.d.ts +8 -0
- package/dist/src/generators/javascript/http.js +183 -0
- package/dist/src/generators/javascript/http.js.map +1 -0
- package/dist/src/generators/javascript/javascript.d.ts +37 -0
- package/dist/src/generators/javascript/javascript.js +676 -0
- package/dist/src/generators/javascript/javascript.js.map +1 -0
- package/dist/src/generators/javascript/jquery.d.ts +15 -0
- package/dist/src/generators/javascript/jquery.js +297 -0
- package/dist/src/generators/javascript/jquery.js.map +1 -0
- package/dist/src/generators/javascript/ky.d.ts +7 -0
- package/dist/src/generators/javascript/ky.js +245 -0
- package/dist/src/generators/javascript/ky.js.map +1 -0
- package/dist/src/generators/javascript/request.d.ts +5 -0
- package/dist/src/generators/javascript/request.js +113 -0
- package/dist/src/generators/javascript/request.js.map +1 -0
- package/dist/src/generators/javascript/superagent.d.ts +10 -0
- package/dist/src/generators/javascript/superagent.js +301 -0
- package/dist/src/generators/javascript/superagent.js.map +1 -0
- package/dist/src/generators/javascript/xhr.d.ts +8 -0
- package/dist/src/generators/javascript/xhr.js +147 -0
- package/dist/src/generators/javascript/xhr.js.map +1 -0
- package/dist/src/generators/json.d.ts +45 -0
- package/dist/src/generators/json.js +197 -0
- package/dist/src/generators/json.js.map +1 -0
- package/dist/src/generators/julia.d.ts +8 -0
- package/dist/src/generators/julia.js +365 -0
- package/dist/src/generators/julia.js.map +1 -0
- package/dist/src/generators/kotlin.d.ts +8 -0
- package/dist/src/generators/kotlin.js +325 -0
- package/dist/src/generators/kotlin.js.map +1 -0
- package/dist/src/generators/lua.d.ts +5 -0
- package/dist/src/generators/lua.js +192 -0
- package/dist/src/generators/lua.js.map +1 -0
- package/dist/src/generators/matlab/common.d.ts +16 -0
- package/dist/src/generators/matlab/common.js +238 -0
- package/dist/src/generators/matlab/common.js.map +1 -0
- package/dist/src/generators/matlab/httpinterface.d.ts +2 -0
- package/dist/src/generators/matlab/httpinterface.js +211 -0
- package/dist/src/generators/matlab/httpinterface.js.map +1 -0
- package/dist/src/generators/matlab/matlab.d.ts +5 -0
- package/dist/src/generators/matlab/matlab.js +29 -0
- package/dist/src/generators/matlab/matlab.js.map +1 -0
- package/dist/src/generators/matlab/webservices.d.ts +2 -0
- package/dist/src/generators/matlab/webservices.js +191 -0
- package/dist/src/generators/matlab/webservices.js.map +1 -0
- package/dist/src/generators/objectivec.d.ts +8 -0
- package/dist/src/generators/objectivec.js +269 -0
- package/dist/src/generators/objectivec.js.map +1 -0
- package/dist/src/generators/ocaml.d.ts +8 -0
- package/dist/src/generators/ocaml.js +150 -0
- package/dist/src/generators/ocaml.js.map +1 -0
- package/dist/src/generators/perl.d.ts +9 -0
- package/dist/src/generators/perl.js +209 -0
- package/dist/src/generators/perl.js.map +1 -0
- package/dist/src/generators/php/guzzle.d.ts +5 -0
- package/dist/src/generators/php/guzzle.js +328 -0
- package/dist/src/generators/php/guzzle.js.map +1 -0
- package/dist/src/generators/php/php.d.ts +8 -0
- package/dist/src/generators/php/php.js +217 -0
- package/dist/src/generators/php/php.js.map +1 -0
- package/dist/src/generators/php/requests.d.ts +5 -0
- package/dist/src/generators/php/requests.js +89 -0
- package/dist/src/generators/php/requests.js.map +1 -0
- package/dist/src/generators/powershell.d.ts +11 -0
- package/dist/src/generators/powershell.js +377 -0
- package/dist/src/generators/powershell.js.map +1 -0
- package/dist/src/generators/python/http.d.ts +5 -0
- package/dist/src/generators/python/http.js +128 -0
- package/dist/src/generators/python/http.js.map +1 -0
- package/dist/src/generators/python/python.d.ts +22 -0
- package/dist/src/generators/python/python.js +1693 -0
- package/dist/src/generators/python/python.js.map +1 -0
- package/dist/src/generators/r/httr.d.ts +10 -0
- package/dist/src/generators/r/httr.js +298 -0
- package/dist/src/generators/r/httr.js.map +1 -0
- package/dist/src/generators/r/httr2.d.ts +5 -0
- package/dist/src/generators/r/httr2.js +214 -0
- package/dist/src/generators/r/httr2.js.map +1 -0
- package/dist/src/generators/ruby/httparty.d.ts +6 -0
- package/dist/src/generators/ruby/httparty.js +308 -0
- package/dist/src/generators/ruby/httparty.js.map +1 -0
- package/dist/src/generators/ruby/ruby.d.ts +14 -0
- package/dist/src/generators/ruby/ruby.js +515 -0
- package/dist/src/generators/ruby/ruby.js.map +1 -0
- package/dist/src/generators/rust.d.ts +8 -0
- package/dist/src/generators/rust.js +177 -0
- package/dist/src/generators/rust.js.map +1 -0
- package/dist/src/generators/swift.d.ts +8 -0
- package/dist/src/generators/swift.js +219 -0
- package/dist/src/generators/swift.js.map +1 -0
- package/dist/src/generators/wget.d.ts +8 -0
- package/dist/src/generators/wget.js +472 -0
- package/dist/src/generators/wget.js.map +1 -0
- package/dist/src/index.d.ts +53 -0
- package/dist/src/index.js +54 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/parse.d.ts +13 -0
- package/dist/src/parse.js +43 -0
- package/dist/src/parse.js.map +1 -0
- package/dist/src/shell/Parser.d.ts +4 -0
- package/dist/src/shell/Parser.js +6 -0
- package/dist/src/shell/Parser.js.map +1 -0
- package/dist/src/shell/Word.d.ts +48 -0
- package/dist/src/shell/Word.js +486 -0
- package/dist/src/shell/Word.js.map +1 -0
- package/dist/src/shell/tokenizer.d.ts +3 -0
- package/dist/src/shell/tokenizer.js +610 -0
- package/dist/src/shell/tokenizer.js.map +1 -0
- package/dist/src/shell/webParser.d.ts +4 -0
- package/dist/src/shell/webParser.js +15 -0
- package/dist/src/shell/webParser.js.map +1 -0
- package/dist/src/utils.d.ts +6 -0
- package/dist/src/utils.js +21 -0
- package/dist/src/utils.js.map +1 -0
- package/dist/tools/compare-requests.d.ts +2 -0
- package/dist/tools/compare-requests.js +448 -0
- package/dist/tools/compare-requests.js.map +1 -0
- package/dist/tools/gen-test.d.ts +2 -0
- package/dist/tools/gen-test.js +114 -0
- package/dist/tools/gen-test.js.map +1 -0
- package/dist/tree-sitter-bash.wasm +0 -0
- package/index.d.ts +3 -0
- package/package.json +97 -0
- package/src/Headers.ts +334 -0
- package/src/Query.ts +147 -0
- package/src/Request.ts +1649 -0
- package/src/Warnings.ts +154 -0
- package/src/cli.ts +561 -0
- package/src/curl/auth.ts +45 -0
- package/src/curl/form.ts +297 -0
- package/src/curl/opts.ts +1417 -0
- package/src/curl/url.ts +241 -0
- package/src/generators/ansible.ts +364 -0
- package/src/generators/c.ts +1371 -0
- package/src/generators/cfml.ts +155 -0
- package/src/generators/clojure.ts +521 -0
- package/src/generators/csharp.ts +383 -0
- package/src/generators/dart.ts +293 -0
- package/src/generators/elixir.ts +349 -0
- package/src/generators/go.ts +269 -0
- package/src/generators/har.ts +147 -0
- package/src/generators/http.ts +194 -0
- package/src/generators/httpie.ts +506 -0
- package/src/generators/java/httpurlconnection.ts +151 -0
- package/src/generators/java/java.ts +280 -0
- package/src/generators/java/jsoup.ts +172 -0
- package/src/generators/java/okhttp.ts +257 -0
- package/src/generators/javascript/axios.ts +395 -0
- package/src/generators/javascript/got.ts +362 -0
- package/src/generators/javascript/http.ts +232 -0
- package/src/generators/javascript/javascript.ts +825 -0
- package/src/generators/javascript/jquery.ts +360 -0
- package/src/generators/javascript/ky.ts +315 -0
- package/src/generators/javascript/request.ts +152 -0
- package/src/generators/javascript/superagent.ts +373 -0
- package/src/generators/javascript/xhr.ts +192 -0
- package/src/generators/json.ts +293 -0
- package/src/generators/julia.ts +398 -0
- package/src/generators/kotlin.ts +355 -0
- package/src/generators/lua.ts +209 -0
- package/src/generators/matlab/common.ts +281 -0
- package/src/generators/matlab/httpinterface.ts +284 -0
- package/src/generators/matlab/matlab.ts +39 -0
- package/src/generators/matlab/webservices.ts +263 -0
- package/src/generators/objectivec.ts +297 -0
- package/src/generators/ocaml.ts +168 -0
- package/src/generators/perl.ts +237 -0
- package/src/generators/php/guzzle.ts +368 -0
- package/src/generators/php/php.ts +239 -0
- package/src/generators/php/requests.ts +103 -0
- package/src/generators/powershell.ts +430 -0
- package/src/generators/python/http.ts +167 -0
- package/src/generators/python/python.ts +1934 -0
- package/src/generators/r/httr.ts +328 -0
- package/src/generators/r/httr2.ts +268 -0
- package/src/generators/ruby/httparty.ts +351 -0
- package/src/generators/ruby/ruby.ts +564 -0
- package/src/generators/rust.ts +243 -0
- package/src/generators/swift.ts +235 -0
- package/src/generators/wget.ts +533 -0
- package/src/index.ts +89 -0
- package/src/parse.ts +72 -0
- package/src/shell/Parser.ts +8 -0
- package/src/shell/Word.ts +531 -0
- package/src/shell/tokenizer.ts +736 -0
- package/src/shell/webParser.ts +18 -0
- package/src/utils.ts +25 -0
- package/tools/compare-requests.ts +571 -0
- package/tools/extract_curl_args.py +511 -0
- package/tools/gen-test.ts +129 -0
|
@@ -0,0 +1,1934 @@
|
|
|
1
|
+
import { CCError, has, UTF8encoder } from "../../utils.js";
|
|
2
|
+
import { Word, eq } from "../../shell/Word.js";
|
|
3
|
+
import { parse, COMMON_SUPPORTED_ARGS } from "../../parse.js";
|
|
4
|
+
import type { Request, Warnings } from "../../parse.js";
|
|
5
|
+
import { Headers } from "../../Headers.js";
|
|
6
|
+
import { wordDecodeURIComponent, percentEncode } from "../../Query.js";
|
|
7
|
+
import { DataParam } from "../../Request.js";
|
|
8
|
+
|
|
9
|
+
import {
|
|
10
|
+
parse as jsonParseLossless,
|
|
11
|
+
stringify as jsonStringifyLossless,
|
|
12
|
+
isSafeNumber,
|
|
13
|
+
isInteger,
|
|
14
|
+
isLosslessNumber,
|
|
15
|
+
} from "lossless-json";
|
|
16
|
+
|
|
17
|
+
// TODO: partiallySupportedArgs
|
|
18
|
+
export const supportedArgs = new Set([
|
|
19
|
+
...COMMON_SUPPORTED_ARGS,
|
|
20
|
+
|
|
21
|
+
"compressed",
|
|
22
|
+
// "no-compressed",
|
|
23
|
+
|
|
24
|
+
// "anyauth",
|
|
25
|
+
// "no-anyauth",
|
|
26
|
+
"digest",
|
|
27
|
+
"no-digest",
|
|
28
|
+
"aws-sigv4",
|
|
29
|
+
"negotiate",
|
|
30
|
+
"no-negotiate",
|
|
31
|
+
"delegation", // GSS/kerberos
|
|
32
|
+
// "service-name", // GSS/kerberos, not supported
|
|
33
|
+
"ntlm",
|
|
34
|
+
"no-ntlm",
|
|
35
|
+
"ntlm-wb",
|
|
36
|
+
"no-ntlm-wb",
|
|
37
|
+
|
|
38
|
+
"http1.1",
|
|
39
|
+
// These are not supported, just better warning message
|
|
40
|
+
"http2",
|
|
41
|
+
"http2-prior-knowledge",
|
|
42
|
+
"http3",
|
|
43
|
+
"http3-only",
|
|
44
|
+
|
|
45
|
+
"cookie-jar",
|
|
46
|
+
|
|
47
|
+
"cert",
|
|
48
|
+
"cacert",
|
|
49
|
+
"key",
|
|
50
|
+
"capath",
|
|
51
|
+
|
|
52
|
+
"form",
|
|
53
|
+
"form-string",
|
|
54
|
+
|
|
55
|
+
"location",
|
|
56
|
+
"no-location",
|
|
57
|
+
"location-trusted", // not exactly supported, just better warning message
|
|
58
|
+
"no-location-trusted",
|
|
59
|
+
"max-redirs",
|
|
60
|
+
|
|
61
|
+
"max-time",
|
|
62
|
+
"connect-timeout",
|
|
63
|
+
|
|
64
|
+
"insecure",
|
|
65
|
+
"no-insecure",
|
|
66
|
+
|
|
67
|
+
"output",
|
|
68
|
+
"upload-file",
|
|
69
|
+
|
|
70
|
+
"next",
|
|
71
|
+
|
|
72
|
+
"proxy",
|
|
73
|
+
"proxy-user",
|
|
74
|
+
]);
|
|
75
|
+
|
|
76
|
+
// https://peps.python.org/pep-3138/
|
|
77
|
+
// https://www.unicode.org/reports/tr44/#GC_Values_Table
|
|
78
|
+
// https://unicode.org/Public/UNIDATA/UnicodeData.txt
|
|
79
|
+
// https://en.wikipedia.org/wiki/Plane_(Unicode)#Overview
|
|
80
|
+
const regexSingleEscape = /'|\\|\p{C}|[^ \P{Z}]/gu;
|
|
81
|
+
const regexDoubleEscape = /"|\\|\p{C}|[^ \P{Z}]/gu;
|
|
82
|
+
|
|
83
|
+
// Also used for Go and R
|
|
84
|
+
export function reprStr(s: string, quote?: '"' | "'"): string {
|
|
85
|
+
if (quote === undefined) {
|
|
86
|
+
quote = "'";
|
|
87
|
+
if (s.includes("'") && !s.includes('"')) {
|
|
88
|
+
quote = '"';
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const regex = quote === "'" ? regexSingleEscape : regexDoubleEscape;
|
|
92
|
+
|
|
93
|
+
return (
|
|
94
|
+
quote +
|
|
95
|
+
s.replace(regex, (c: string): string => {
|
|
96
|
+
switch (c) {
|
|
97
|
+
case "\x07":
|
|
98
|
+
return "\\a";
|
|
99
|
+
case "\b":
|
|
100
|
+
return "\\b";
|
|
101
|
+
case "\f":
|
|
102
|
+
return "\\f";
|
|
103
|
+
case "\n":
|
|
104
|
+
return "\\n";
|
|
105
|
+
case "\r":
|
|
106
|
+
return "\\r";
|
|
107
|
+
case "\t":
|
|
108
|
+
return "\\t";
|
|
109
|
+
case "\v":
|
|
110
|
+
return "\\v";
|
|
111
|
+
case "\\":
|
|
112
|
+
return "\\\\";
|
|
113
|
+
case "'":
|
|
114
|
+
return "\\'";
|
|
115
|
+
case '"':
|
|
116
|
+
return '\\"';
|
|
117
|
+
}
|
|
118
|
+
const hex = (c.codePointAt(0) as number).toString(16);
|
|
119
|
+
if (hex.length <= 2) {
|
|
120
|
+
return "\\x" + hex.padStart(2, "0");
|
|
121
|
+
}
|
|
122
|
+
if (hex.length <= 4) {
|
|
123
|
+
return "\\u" + hex.padStart(4, "0");
|
|
124
|
+
}
|
|
125
|
+
return "\\U" + hex.padStart(8, "0");
|
|
126
|
+
}) +
|
|
127
|
+
quote
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// TODO: use this if string contains unmatched surrogates?
|
|
132
|
+
// It just replaces them with the replacement character, but at least that code would run.
|
|
133
|
+
export function pybescComplex(s: string): string {
|
|
134
|
+
let quote = "'";
|
|
135
|
+
if (s.includes("'") && !s.includes('"')) {
|
|
136
|
+
quote = '"';
|
|
137
|
+
}
|
|
138
|
+
const quoteCode = quote.charCodeAt(0);
|
|
139
|
+
|
|
140
|
+
// TODO: using UTF-8 here is overly simplistic and how to encode here
|
|
141
|
+
// is a pretty complicated decision.
|
|
142
|
+
// For starters, it would be more correct to use the same encoding as
|
|
143
|
+
// the terminal when running from the command line.
|
|
144
|
+
const bytes = UTF8encoder.encode(s);
|
|
145
|
+
|
|
146
|
+
return (
|
|
147
|
+
"b" +
|
|
148
|
+
quote +
|
|
149
|
+
[...bytes]
|
|
150
|
+
.map((c: number): string => {
|
|
151
|
+
switch (c) {
|
|
152
|
+
case 0x07:
|
|
153
|
+
return "\\a";
|
|
154
|
+
case 0x08:
|
|
155
|
+
return "\\b";
|
|
156
|
+
case 0x0c:
|
|
157
|
+
return "\\f";
|
|
158
|
+
case 0x0a:
|
|
159
|
+
return "\\n";
|
|
160
|
+
case 0x0d:
|
|
161
|
+
return "\\r";
|
|
162
|
+
case 0x09:
|
|
163
|
+
return "\\t";
|
|
164
|
+
case 0x0b:
|
|
165
|
+
return "\\v";
|
|
166
|
+
case 0x5c:
|
|
167
|
+
return "\\\\";
|
|
168
|
+
case quoteCode:
|
|
169
|
+
return "\\" + String.fromCharCode(c);
|
|
170
|
+
}
|
|
171
|
+
if (c >= 0x20 && c < 0x7f) {
|
|
172
|
+
return String.fromCharCode(c);
|
|
173
|
+
}
|
|
174
|
+
const hex = c.toString(16);
|
|
175
|
+
return "\\x" + hex.padStart(2, "0");
|
|
176
|
+
})
|
|
177
|
+
.join("") +
|
|
178
|
+
quote
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export function reprStrBinary(s: string): string {
|
|
183
|
+
const sEsc = reprStr(s);
|
|
184
|
+
// We check until 0x7F instead of 0xFF because curl (running in a UTF-8 terminal) when it gets
|
|
185
|
+
// bytes sends them as is, but if we pass b'\x80' to Requests, it will encode that byte as
|
|
186
|
+
// Latin-1 (presumably for backwards compatibility) instead of UTF-8.
|
|
187
|
+
if (/^[\x00-\x7f]*$/.test(s)) {
|
|
188
|
+
return "b" + sEsc;
|
|
189
|
+
}
|
|
190
|
+
// TODO: unmatched surrogates will generate code that throws an error
|
|
191
|
+
// e.g.: '\uDC00'.encode()
|
|
192
|
+
return sEsc + ".encode()";
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export type OSVars = { [key: string]: string };
|
|
196
|
+
export function repr(
|
|
197
|
+
word: Word,
|
|
198
|
+
osVars: OSVars,
|
|
199
|
+
imports: Set<string>,
|
|
200
|
+
binary = false,
|
|
201
|
+
// os.getenv('MYVAR') returns None if MYVAR is not set
|
|
202
|
+
// os.getenv('MYVAR', '') returns '' if MYVAR is not set but it's a bit more verbose,
|
|
203
|
+
// so setting errorOk to true will use the shorter version
|
|
204
|
+
errorOk = false,
|
|
205
|
+
): string {
|
|
206
|
+
const reprFn = binary ? reprStrBinary : reprStr;
|
|
207
|
+
const reprs = [];
|
|
208
|
+
for (const t of word.tokens) {
|
|
209
|
+
if (typeof t === "string") {
|
|
210
|
+
reprs.push(reprFn(t));
|
|
211
|
+
} else if (t.type === "variable") {
|
|
212
|
+
// TODO: getenvb() is not available on Windows
|
|
213
|
+
const fn = binary ? "os.getenvb" : "os.getenv";
|
|
214
|
+
let getEnvCall = fn + "(" + reprFn(t.value);
|
|
215
|
+
if (!errorOk || word.tokens.length > 1) {
|
|
216
|
+
getEnvCall += ", " + reprFn("");
|
|
217
|
+
}
|
|
218
|
+
getEnvCall += ")";
|
|
219
|
+
reprs.push(getEnvCall);
|
|
220
|
+
// TODO: if the result of a repr() is discarded, this adds an unused import
|
|
221
|
+
imports.add("os");
|
|
222
|
+
} else if (t.type === "command") {
|
|
223
|
+
// TODO: warn that shell=True is a bad idea
|
|
224
|
+
// or properly parse the subcommand and render it as an array
|
|
225
|
+
let subprocessCall =
|
|
226
|
+
"subprocess.run(" +
|
|
227
|
+
reprStr(t.value) +
|
|
228
|
+
", shell=True, capture_output=True";
|
|
229
|
+
if (!binary) {
|
|
230
|
+
subprocessCall += ", text=True";
|
|
231
|
+
}
|
|
232
|
+
subprocessCall += ").stdout";
|
|
233
|
+
|
|
234
|
+
// TODO: generate a descriptive command name with ChatGPT
|
|
235
|
+
// TODO: if there's only one command, name the variable "command" instead of "command1"
|
|
236
|
+
let i = 1;
|
|
237
|
+
let pyVar = "command" + i;
|
|
238
|
+
// We need to check because we often try to represent the same
|
|
239
|
+
// token twice and discard one of the attempts.
|
|
240
|
+
// This is linear time but hopefully there's not that many subcommands.
|
|
241
|
+
while (pyVar in osVars && osVars[pyVar] !== subprocessCall) {
|
|
242
|
+
i++;
|
|
243
|
+
pyVar = "command" + i;
|
|
244
|
+
if (i > Number.MAX_SAFE_INTEGER) {
|
|
245
|
+
throw new CCError("lol");
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
osVars[pyVar] = subprocessCall;
|
|
249
|
+
reprs.push(pyVar);
|
|
250
|
+
// TODO: if the result of a repr() is discarded, this adds an unused import
|
|
251
|
+
imports.add("subprocess");
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return reprs.join(" + ");
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function reprb(word: Word, osVars: OSVars, imports: Set<string>): string {
|
|
258
|
+
return repr(word, osVars, imports, true);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
export function asFloat(
|
|
262
|
+
word: Word,
|
|
263
|
+
osVars: OSVars,
|
|
264
|
+
imports: Set<string>,
|
|
265
|
+
): string {
|
|
266
|
+
if (word.isString()) {
|
|
267
|
+
// TODO: check it's actually a valid float
|
|
268
|
+
return word.toString();
|
|
269
|
+
}
|
|
270
|
+
return "float(" + repr(word, osVars, imports, false, true) + ")";
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export function asInt(
|
|
274
|
+
word: Word,
|
|
275
|
+
osVars: OSVars,
|
|
276
|
+
imports: Set<string>,
|
|
277
|
+
): string {
|
|
278
|
+
if (word.isString()) {
|
|
279
|
+
// TODO: check it's actually a valid int
|
|
280
|
+
return word.toString();
|
|
281
|
+
}
|
|
282
|
+
return "int(" + repr(word, osVars, imports, false, true) + ")";
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Port of Python's json.dumps() with its default options, which is what Requests uses
|
|
286
|
+
// https://github.com/psf/requests/blob/b0e025ade7ed30ed53ab61f542779af7e024932e/requests/models.py#L473
|
|
287
|
+
// It's different from JSON.stringify(). Namely, it adds spaces after ',' and ':'
|
|
288
|
+
// and all non-ASCII characters in strings are escaped:
|
|
289
|
+
// > JSON.stringify('\xFF')
|
|
290
|
+
// '"ÿ"'
|
|
291
|
+
// >>> json.dumps('\xFF')
|
|
292
|
+
// '"\\u00ff"'
|
|
293
|
+
const pythonJsonEscape = /"|\\|[^\x20-\x7E]/g;
|
|
294
|
+
function jsonRepr(s: string): string {
|
|
295
|
+
return (
|
|
296
|
+
'"' +
|
|
297
|
+
s.replace(pythonJsonEscape, (c: string): string => {
|
|
298
|
+
// https://tc39.es/ecma262/#table-json-single-character-escapes
|
|
299
|
+
switch (c) {
|
|
300
|
+
case "\b":
|
|
301
|
+
return "\\b";
|
|
302
|
+
case "\t":
|
|
303
|
+
return "\\t";
|
|
304
|
+
case "\n":
|
|
305
|
+
return "\\n";
|
|
306
|
+
case "\f":
|
|
307
|
+
return "\\f";
|
|
308
|
+
case "\r":
|
|
309
|
+
return "\\r";
|
|
310
|
+
case "\\":
|
|
311
|
+
return "\\\\";
|
|
312
|
+
case '"':
|
|
313
|
+
return '\\"';
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const hex = c.charCodeAt(0).toString(16);
|
|
317
|
+
return "\\u" + hex.padStart(4, "0");
|
|
318
|
+
}) +
|
|
319
|
+
'"'
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
function ensure_minimum_exponent_length(n: string): string {
|
|
324
|
+
// If there's only 1 exponent digit, add a leading 0 to it
|
|
325
|
+
// ensure_minimum_exponent_length('1e-7') => '1e-07'
|
|
326
|
+
const [mantissa, exponent] = n.split("e");
|
|
327
|
+
const exponentSign = exponent[0];
|
|
328
|
+
const exponentValue = exponent.slice(1);
|
|
329
|
+
if (exponentValue.length === 1) {
|
|
330
|
+
return mantissa + "e" + exponentSign + "0" + exponentValue;
|
|
331
|
+
}
|
|
332
|
+
return n;
|
|
333
|
+
}
|
|
334
|
+
function floatAsPython(value: number): string {
|
|
335
|
+
// JSON.stringify() and lossless-json's stringify() don't stringify floats like Python.
|
|
336
|
+
// Notably, JavaScript doesn't add a trailing '.0' to floats that are integers but Python does
|
|
337
|
+
// JSON.stringify(10.0) => '10'
|
|
338
|
+
// str(10.0) => '10.0'
|
|
339
|
+
//
|
|
340
|
+
// Python adds a leading 0 to exponent notation numbers with 1 exponent digit
|
|
341
|
+
// JSON.stringify(1e-7) => '1e-7'
|
|
342
|
+
// str(1e-7) => '1e-07'
|
|
343
|
+
//
|
|
344
|
+
// Finally, Python will switch to scientific notation if the number has more than
|
|
345
|
+
// 17 digits not in scientific notation.
|
|
346
|
+
//
|
|
347
|
+
// Python's float formatting starts here:
|
|
348
|
+
// https://github.com/python/cpython/blob/bdc93b8a3563b4a3adb25fa902c0c879ccf427f6/Python/pystrtod.c#L915-L918
|
|
349
|
+
// and is ultimately this code:
|
|
350
|
+
// snprintf(buf, buf_size, "%.17g", val)
|
|
351
|
+
// change_decimal_from_locale_to_dot(buffer); // not important
|
|
352
|
+
// ensure_minimum_exponent_length(buffer, buf_size);
|
|
353
|
+
// ensure_decimal_point(buffer, buf_size, 17); // can switch to exponent notation
|
|
354
|
+
//
|
|
355
|
+
// And JavaScript's formatting is specified here:
|
|
356
|
+
// https://tc39.es/ecma262/multipage/ecmascript-data-types-and-values.html#sec-numeric-types-number-tostring
|
|
357
|
+
let asJsStr = value.toString();
|
|
358
|
+
if (asJsStr.includes("e")) {
|
|
359
|
+
asJsStr = ensure_minimum_exponent_length(asJsStr);
|
|
360
|
+
} else {
|
|
361
|
+
if (isInteger(asJsStr)) {
|
|
362
|
+
asJsStr += ".0";
|
|
363
|
+
}
|
|
364
|
+
// If there's more than 17 digits of precision, switch to scientific notation
|
|
365
|
+
const significantDigits = asJsStr
|
|
366
|
+
.replace(/^-/, "")
|
|
367
|
+
.replace(".", "")
|
|
368
|
+
.replace(/^0+/, "");
|
|
369
|
+
const asExponential = ensure_minimum_exponent_length(value.toExponential());
|
|
370
|
+
if (
|
|
371
|
+
significantDigits.length > 17 ||
|
|
372
|
+
(asExponential.length < asJsStr.length &&
|
|
373
|
+
asJsStr.split(".")[1].length > 4)
|
|
374
|
+
) {
|
|
375
|
+
asJsStr = asExponential;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return asJsStr;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function jsonDumps(obj: string | number | boolean | object | null): string {
|
|
382
|
+
if (isLosslessNumber(obj)) {
|
|
383
|
+
const numAsStr = jsonStringifyLossless(obj) as string;
|
|
384
|
+
if (isInteger(numAsStr)) {
|
|
385
|
+
return numAsStr;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
if (!isSafeNumber(numAsStr)) {
|
|
389
|
+
throw new CCError("float unrepresentable in Python: " + numAsStr);
|
|
390
|
+
}
|
|
391
|
+
// Can't be bigint because it's !isInteger and isSafeNumber
|
|
392
|
+
return floatAsPython(obj.valueOf() as number);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
switch (typeof obj) {
|
|
396
|
+
case "string":
|
|
397
|
+
return jsonRepr(obj);
|
|
398
|
+
case "number":
|
|
399
|
+
// If the number in the JSON file is very large, it'll turn into Infinity
|
|
400
|
+
if (!isFinite(obj)) {
|
|
401
|
+
throw new CCError("found Infitiny in JSON");
|
|
402
|
+
}
|
|
403
|
+
// TODO: If the number in the JSON file is too big for JavaScript, we will lose information
|
|
404
|
+
// TODO: JavaScript and Python serialize floats differently.
|
|
405
|
+
// JSON.stringify(2e2) => 200
|
|
406
|
+
// json.dumps(2e2) => 200.0
|
|
407
|
+
return obj.toString();
|
|
408
|
+
case "boolean":
|
|
409
|
+
return obj.toString();
|
|
410
|
+
case "object":
|
|
411
|
+
if (obj === null) {
|
|
412
|
+
return "null";
|
|
413
|
+
}
|
|
414
|
+
if (Array.isArray(obj)) {
|
|
415
|
+
return "[" + obj.map(jsonDumps).join(", ") + "]";
|
|
416
|
+
}
|
|
417
|
+
return (
|
|
418
|
+
"{" +
|
|
419
|
+
Object.entries(obj)
|
|
420
|
+
.map((e) => jsonRepr(e[0]) + ": " + jsonDumps(e[1]))
|
|
421
|
+
.join(", ") +
|
|
422
|
+
"}"
|
|
423
|
+
);
|
|
424
|
+
default:
|
|
425
|
+
throw new CCError(
|
|
426
|
+
"unexpected object type that shouldn't appear in JSON: " + typeof obj,
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function objToPython(
|
|
432
|
+
obj: string | number | boolean | object | null,
|
|
433
|
+
indent = 0,
|
|
434
|
+
): string {
|
|
435
|
+
if (isLosslessNumber(obj)) {
|
|
436
|
+
const numAsStr = jsonStringifyLossless(obj) as string;
|
|
437
|
+
// If the number is a large float, it might not be representable in Python
|
|
438
|
+
// Both JavaScript and Python use f64 so we check if the float
|
|
439
|
+
// is representable in JavaScript.
|
|
440
|
+
if (!isInteger(numAsStr) && !isSafeNumber(numAsStr)) {
|
|
441
|
+
throw new CCError("float unrepresentable in Python: " + numAsStr);
|
|
442
|
+
}
|
|
443
|
+
// Displaying floats as they will be serialized in Python would help users
|
|
444
|
+
// understand why they're getting the "JSON won't be serialized as it was originally"
|
|
445
|
+
// message, but I think displaying them as they appear in the JSON is likely
|
|
446
|
+
// to be more convenient if you need to edit the value.
|
|
447
|
+
return numAsStr;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
switch (typeof obj) {
|
|
451
|
+
case "string":
|
|
452
|
+
return reprStr(obj);
|
|
453
|
+
case "number":
|
|
454
|
+
// TODO: there are differences in number serialization between Python and JavaScript
|
|
455
|
+
// TODO: if the number in the JSON file is too big for JavaScript, we will lose information
|
|
456
|
+
return obj.toString();
|
|
457
|
+
case "boolean":
|
|
458
|
+
return obj ? "True" : "False";
|
|
459
|
+
case "object":
|
|
460
|
+
if (obj === null) {
|
|
461
|
+
return "None";
|
|
462
|
+
}
|
|
463
|
+
if (Array.isArray(obj)) {
|
|
464
|
+
if (obj.length === 0) {
|
|
465
|
+
return "[]";
|
|
466
|
+
}
|
|
467
|
+
let s = "[\n";
|
|
468
|
+
for (const item of obj) {
|
|
469
|
+
s += " ".repeat(indent + 4) + objToPython(item, indent + 4) + ",\n";
|
|
470
|
+
}
|
|
471
|
+
s += " ".repeat(indent) + "]";
|
|
472
|
+
return s;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
if (Object.keys(obj).length === 0) {
|
|
476
|
+
return "{}";
|
|
477
|
+
}
|
|
478
|
+
{
|
|
479
|
+
let s = "{\n";
|
|
480
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
481
|
+
// repr() because JSON keys must be strings.
|
|
482
|
+
s +=
|
|
483
|
+
" ".repeat(indent + 4) +
|
|
484
|
+
reprStr(k) +
|
|
485
|
+
": " +
|
|
486
|
+
objToPython(v, indent + 4) +
|
|
487
|
+
",\n";
|
|
488
|
+
}
|
|
489
|
+
s += " ".repeat(indent) + "}";
|
|
490
|
+
return s;
|
|
491
|
+
}
|
|
492
|
+
default:
|
|
493
|
+
throw new CCError(
|
|
494
|
+
"unexpected object type that shouldn't appear in JSON: " + typeof obj,
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
export function formatHeaders(
|
|
500
|
+
headers: Headers,
|
|
501
|
+
commentedOutHeaders: { [key: string]: string },
|
|
502
|
+
osVars: OSVars,
|
|
503
|
+
imports: Set<string>,
|
|
504
|
+
): string {
|
|
505
|
+
// TODO: what if there are repeat headers
|
|
506
|
+
let headerDict = "headers = {\n";
|
|
507
|
+
for (const [headerName, headerValue] of headers) {
|
|
508
|
+
if (headerValue === null) {
|
|
509
|
+
continue;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
let lineStart;
|
|
513
|
+
const headerNameLower = headerName.toLowerCase().toString();
|
|
514
|
+
if (has(commentedOutHeaders, headerNameLower)) {
|
|
515
|
+
if (commentedOutHeaders[headerNameLower]) {
|
|
516
|
+
headerDict += " # " + commentedOutHeaders[headerNameLower] + "\n";
|
|
517
|
+
}
|
|
518
|
+
lineStart = " # ";
|
|
519
|
+
} else {
|
|
520
|
+
lineStart = " ";
|
|
521
|
+
}
|
|
522
|
+
headerDict +=
|
|
523
|
+
lineStart +
|
|
524
|
+
repr(headerName, osVars, imports) +
|
|
525
|
+
": " +
|
|
526
|
+
repr(headerValue, osVars, imports) +
|
|
527
|
+
",\n";
|
|
528
|
+
}
|
|
529
|
+
headerDict += "}\n";
|
|
530
|
+
return headerDict;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
function decodePercentEncoding(s: Word): Word | null {
|
|
534
|
+
let decoded;
|
|
535
|
+
try {
|
|
536
|
+
// https://url.spec.whatwg.org/#urlencoded-parsing recommends replacing + with space
|
|
537
|
+
// before decoding.
|
|
538
|
+
decoded = wordDecodeURIComponent(s.replace(/\+/g, " "));
|
|
539
|
+
} catch (e) {
|
|
540
|
+
if (e instanceof URIError) {
|
|
541
|
+
// String contains invalid percent encoded characters
|
|
542
|
+
return null;
|
|
543
|
+
}
|
|
544
|
+
throw e;
|
|
545
|
+
}
|
|
546
|
+
// If the query string doesn't round-trip, we cannot properly convert it.
|
|
547
|
+
const roundTripKey = percentEncode(decoded);
|
|
548
|
+
// If the original data used %20 instead of + (what requests will send), that's close enough
|
|
549
|
+
if (!eq(roundTripKey, s) && !eq(roundTripKey.replace(/%20/g, "+"), s)) {
|
|
550
|
+
return null;
|
|
551
|
+
}
|
|
552
|
+
return decoded;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function dataEntriesToDict(
|
|
556
|
+
dataEntries: Array<[string, string]>,
|
|
557
|
+
): { [key: string]: Array<string> } | null {
|
|
558
|
+
// Group keys
|
|
559
|
+
// TODO: because keys can be code that reads from a file, those should not be considered the
|
|
560
|
+
// same key, for example what if that file is /dev/urandom.
|
|
561
|
+
// TODO: would we need to distinguish if /dev/urandom came from @/dev/urandom or from @-?
|
|
562
|
+
const asDict: { [key: string]: Array<string> } = {};
|
|
563
|
+
let prevKey = null;
|
|
564
|
+
for (const [key, val] of dataEntries) {
|
|
565
|
+
if (prevKey === key) {
|
|
566
|
+
asDict[key].push(val);
|
|
567
|
+
} else {
|
|
568
|
+
if (!Object.prototype.hasOwnProperty.call(asDict, key)) {
|
|
569
|
+
asDict[key] = [val];
|
|
570
|
+
} else {
|
|
571
|
+
// A repeated key with a different key between one of its repetitions
|
|
572
|
+
// means we can't represent these entries as a dictionary.
|
|
573
|
+
return null;
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
prevKey = key;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
return asDict;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function dataEntriesToPython(dataEntries: Array<[string, string]>): string {
|
|
583
|
+
if (dataEntries.length === 0) {
|
|
584
|
+
return "''"; // This shouldn't happen
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
const entriesDict = dataEntriesToDict(dataEntries);
|
|
588
|
+
if (entriesDict !== null) {
|
|
589
|
+
if (Object.keys(entriesDict).length === 0) {
|
|
590
|
+
return "''"; // This shouldn't happen
|
|
591
|
+
}
|
|
592
|
+
let s = "{\n";
|
|
593
|
+
for (const [key, vals] of Object.entries(entriesDict)) {
|
|
594
|
+
s += " " + key + ": ";
|
|
595
|
+
if (vals.length === 0) {
|
|
596
|
+
s += "''"; // This shouldn't happen
|
|
597
|
+
} else if (vals.length === 1) {
|
|
598
|
+
s += vals[0];
|
|
599
|
+
} else {
|
|
600
|
+
s += "[\n";
|
|
601
|
+
for (const val of vals) {
|
|
602
|
+
s += " " + val + ",\n";
|
|
603
|
+
}
|
|
604
|
+
s += " ]";
|
|
605
|
+
}
|
|
606
|
+
s += ",\n";
|
|
607
|
+
}
|
|
608
|
+
s += "}";
|
|
609
|
+
return s;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
let s = "[\n";
|
|
613
|
+
for (const entry of dataEntries) {
|
|
614
|
+
const [key, val] = entry;
|
|
615
|
+
s += " (" + key + ", " + val + "),\n";
|
|
616
|
+
}
|
|
617
|
+
s += "]";
|
|
618
|
+
return s;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function formatDataAsEntries(
|
|
622
|
+
dataArray: DataParam[],
|
|
623
|
+
osVars: OSVars,
|
|
624
|
+
imports: Set<string>,
|
|
625
|
+
variableName: "data" | "params" = "data",
|
|
626
|
+
): [string, string] | null {
|
|
627
|
+
// This code is more complicated than you might expect because it needs
|
|
628
|
+
// to handle a --data-urlencode that reads from a file followed by --json
|
|
629
|
+
// because --json doesn't add an '&' before its value. Specifically, we
|
|
630
|
+
// have these cases:
|
|
631
|
+
//
|
|
632
|
+
// --data-urlencode @filename --json =value
|
|
633
|
+
// {open('filename').read(): 'value'}
|
|
634
|
+
//
|
|
635
|
+
// --data-urlencode @filename --json key=value
|
|
636
|
+
// {open('filename').read() + 'key': 'value'}
|
|
637
|
+
//
|
|
638
|
+
// --data-urlencode @filename --json key
|
|
639
|
+
// error
|
|
640
|
+
//
|
|
641
|
+
// --data-urlencode name@filename --json value
|
|
642
|
+
// {'name': open('filename').read() + 'value'}
|
|
643
|
+
//
|
|
644
|
+
// --data-urlencode name@filename --json key=value
|
|
645
|
+
// error
|
|
646
|
+
//
|
|
647
|
+
// --data-urlencode name@filename --json =blah
|
|
648
|
+
// error
|
|
649
|
+
//
|
|
650
|
+
// --data-urlencode adds an '&' before its value, so we don't have to worry about
|
|
651
|
+
// --json <foo> --data-urlencode <bar>
|
|
652
|
+
for (const d of dataArray) {
|
|
653
|
+
if (!(d instanceof Word) && d.filetype !== "urlencode") {
|
|
654
|
+
return null;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const dataEntries: Array<[string, string | null]> = [];
|
|
659
|
+
let percentWarn = new Word();
|
|
660
|
+
for (const [i, d] of dataArray.entries()) {
|
|
661
|
+
if (d instanceof Word) {
|
|
662
|
+
let newEntries = d.split("&");
|
|
663
|
+
|
|
664
|
+
const prevEntry = i > 0 ? dataEntries[dataEntries.length - 1] : null;
|
|
665
|
+
if (prevEntry !== null) {
|
|
666
|
+
// If there's a prevEntry, we can assume it came from --data-urlencode
|
|
667
|
+
// because it would be part of the current `d` string if it didn't.
|
|
668
|
+
const [first, ...rest] = newEntries;
|
|
669
|
+
if (first.includes("=") && prevEntry[1] === null) {
|
|
670
|
+
const [key, val] = first.split("=", 2);
|
|
671
|
+
const decodedKey = decodePercentEncoding(key);
|
|
672
|
+
if (decodedKey === null) {
|
|
673
|
+
return null;
|
|
674
|
+
}
|
|
675
|
+
const decodedVal = decodePercentEncoding(val);
|
|
676
|
+
if (decodedVal === null) {
|
|
677
|
+
return null;
|
|
678
|
+
}
|
|
679
|
+
if (key.toBool()) {
|
|
680
|
+
prevEntry[0] += " + " + repr(decodedKey, osVars, imports);
|
|
681
|
+
}
|
|
682
|
+
prevEntry[1] = repr(decodedVal, osVars, imports);
|
|
683
|
+
|
|
684
|
+
if (!percentWarn.toBool()) {
|
|
685
|
+
if (key.includes("%20")) {
|
|
686
|
+
percentWarn = key;
|
|
687
|
+
}
|
|
688
|
+
if (val.includes("%20")) {
|
|
689
|
+
percentWarn = val;
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
} else if (!first.includes("=") && prevEntry[1] !== null) {
|
|
693
|
+
if (first.toBool()) {
|
|
694
|
+
const decodedVal = decodePercentEncoding(first);
|
|
695
|
+
if (decodedVal === null) {
|
|
696
|
+
return null;
|
|
697
|
+
}
|
|
698
|
+
prevEntry[1] += " + " + repr(decodedVal, osVars, imports);
|
|
699
|
+
|
|
700
|
+
if (!percentWarn.toBool() && first.includes("%20")) {
|
|
701
|
+
percentWarn = first;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
} else {
|
|
705
|
+
return null;
|
|
706
|
+
}
|
|
707
|
+
newEntries = rest;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
for (const [j, entry] of newEntries.entries()) {
|
|
711
|
+
if (
|
|
712
|
+
entry.isEmpty() &&
|
|
713
|
+
j === newEntries.length - 1 &&
|
|
714
|
+
i !== dataArray.length - 1
|
|
715
|
+
) {
|
|
716
|
+
// A --data-urlencoded should come next
|
|
717
|
+
continue;
|
|
718
|
+
}
|
|
719
|
+
if (!entry.includes("=")) {
|
|
720
|
+
return null;
|
|
721
|
+
}
|
|
722
|
+
const [key, val] = entry.split("=", 2);
|
|
723
|
+
const decodedKey = decodePercentEncoding(key);
|
|
724
|
+
if (decodedKey === null) {
|
|
725
|
+
return null;
|
|
726
|
+
}
|
|
727
|
+
const decodedVal = decodePercentEncoding(val);
|
|
728
|
+
if (decodedVal === null) {
|
|
729
|
+
return null;
|
|
730
|
+
}
|
|
731
|
+
dataEntries.push([
|
|
732
|
+
repr(decodedKey, osVars, imports),
|
|
733
|
+
repr(decodedVal, osVars, imports),
|
|
734
|
+
]);
|
|
735
|
+
|
|
736
|
+
if (!percentWarn.toBool()) {
|
|
737
|
+
if (key.includes("%20")) {
|
|
738
|
+
percentWarn = key;
|
|
739
|
+
}
|
|
740
|
+
if (val.includes("%20")) {
|
|
741
|
+
percentWarn = val;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
continue;
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
// TODO: I bet Python doesn't treat file paths identically to curl
|
|
750
|
+
const readFile = eq(d.filename, "-")
|
|
751
|
+
? "sys.stdin.read()"
|
|
752
|
+
: "open(" + repr(d.filename, osVars, imports, false, true) + ").read()";
|
|
753
|
+
|
|
754
|
+
if (!d.name) {
|
|
755
|
+
dataEntries.push([readFile, null]);
|
|
756
|
+
} else {
|
|
757
|
+
// Curl doesn't percent encode the name but Requests will
|
|
758
|
+
if (!eq(d.name, percentEncode(d.name))) {
|
|
759
|
+
return null;
|
|
760
|
+
}
|
|
761
|
+
dataEntries.push([repr(d.name, osVars, imports), readFile]);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
if (dataEntries.some((e) => e[1] === null)) {
|
|
766
|
+
return null;
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
return [
|
|
770
|
+
variableName +
|
|
771
|
+
" = " +
|
|
772
|
+
dataEntriesToPython(dataEntries as [string, string][]) +
|
|
773
|
+
"\n",
|
|
774
|
+
percentWarn.toString(),
|
|
775
|
+
];
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function formatDataAsStr(
|
|
779
|
+
dataArray: DataParam[],
|
|
780
|
+
imports: Set<string>,
|
|
781
|
+
osVars: OSVars,
|
|
782
|
+
variableName: "data" | "params" = "data",
|
|
783
|
+
): [string, boolean] {
|
|
784
|
+
// If one of the arguments has to be binary, then they all have to be binary
|
|
785
|
+
// because we can't mix bytes and str.
|
|
786
|
+
// An argument has to be binary when the input command has
|
|
787
|
+
// --data-binary @filename
|
|
788
|
+
// otherwise we could generate code that will try to read an image file as text and error.
|
|
789
|
+
const binary = dataArray.some(
|
|
790
|
+
(d) => !(d instanceof Word) && d.filetype === "binary",
|
|
791
|
+
);
|
|
792
|
+
const reprFunc = binary ? reprb : repr;
|
|
793
|
+
const prefix = binary ? "b" : "";
|
|
794
|
+
const mode = binary ? ", 'rb'" : "";
|
|
795
|
+
|
|
796
|
+
// If we see a string with non-ASCII characters, or read from a file (which may contain
|
|
797
|
+
// non-ASCII characters), we convert the entire string to bytes at the end.
|
|
798
|
+
// curl sends bytes as-is, which is presumably in UTF-8, whereas Requests sends
|
|
799
|
+
// 0x80-0xFF as Latin-1 (as-is) and throws an error if it sees codepoints
|
|
800
|
+
// above 0xFF.
|
|
801
|
+
// TODO: is this actually helpful?
|
|
802
|
+
let encode = false;
|
|
803
|
+
let encodeOnSeparateLine = false;
|
|
804
|
+
|
|
805
|
+
const lines = [];
|
|
806
|
+
|
|
807
|
+
let extra = new Word();
|
|
808
|
+
let i, d;
|
|
809
|
+
for ([i, d] of dataArray.entries()) {
|
|
810
|
+
const op = i === 0 ? "=" : "+=";
|
|
811
|
+
let line = variableName + " " + op + " ";
|
|
812
|
+
|
|
813
|
+
if (i < dataArray.length - 1 && d instanceof Word && d.endsWith("&")) {
|
|
814
|
+
// Put the trailing '&' on the next line so that we don't have single '&'s on their own lines
|
|
815
|
+
extra = new Word("&");
|
|
816
|
+
d = d.slice(0, -1);
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
if (d instanceof Word) {
|
|
820
|
+
if (d.length) {
|
|
821
|
+
line += reprFunc(d, osVars, imports);
|
|
822
|
+
lines.push(line);
|
|
823
|
+
encode ||= d.test(/[^\x00-\x7F]/);
|
|
824
|
+
}
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
const { filetype, name, filename } = d;
|
|
829
|
+
if (filetype === "urlencode" && name) {
|
|
830
|
+
line += reprFunc(extra.add(name).append("="), osVars, imports) + " + ";
|
|
831
|
+
encodeOnSeparateLine = true; // we would need to add parentheses because of the +
|
|
832
|
+
} else if (extra.toBool()) {
|
|
833
|
+
line += reprFunc(extra, osVars, imports) + " + ";
|
|
834
|
+
encodeOnSeparateLine = true;
|
|
835
|
+
}
|
|
836
|
+
if (extra.toBool()) {
|
|
837
|
+
encodeOnSeparateLine = true; // we would need to add parentheses because of the +
|
|
838
|
+
}
|
|
839
|
+
|
|
840
|
+
let readFile = "";
|
|
841
|
+
if (eq(filename, "-")) {
|
|
842
|
+
readFile += binary ? "sys.stdin.buffer" : "sys.stdin";
|
|
843
|
+
imports.add("sys");
|
|
844
|
+
} else {
|
|
845
|
+
// TODO: if filename is a command, this won't work because unlike bash,
|
|
846
|
+
// Python won't remove the trailing newline from the result of a command
|
|
847
|
+
// we need to add .trim()
|
|
848
|
+
line =
|
|
849
|
+
"with open(" +
|
|
850
|
+
repr(filename, osVars, imports, false, true) +
|
|
851
|
+
mode +
|
|
852
|
+
") as f:\n " +
|
|
853
|
+
line;
|
|
854
|
+
readFile += "f";
|
|
855
|
+
}
|
|
856
|
+
readFile += ".read()";
|
|
857
|
+
if (!["binary", "json", "urlencode"].includes(filetype)) {
|
|
858
|
+
readFile += `.replace(${prefix}'\\n', ${prefix}'').replace(${prefix}'\\r', ${prefix}'')`;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
if (filetype === "urlencode") {
|
|
862
|
+
readFile = "quote_plus(" + readFile + ")";
|
|
863
|
+
if (binary) {
|
|
864
|
+
// quote_plus() always returns a string
|
|
865
|
+
readFile += ".encode()";
|
|
866
|
+
}
|
|
867
|
+
imports.add("urllib.parse.quote_plus");
|
|
868
|
+
} else {
|
|
869
|
+
// --data-urlencode files don't need to be encoded because
|
|
870
|
+
// they'll be percent-encoded and therefore ASCII-only
|
|
871
|
+
encode = true;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
line += readFile;
|
|
875
|
+
lines.push(line);
|
|
876
|
+
extra = new Word();
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
if (binary) {
|
|
880
|
+
encode = false;
|
|
881
|
+
} else if (encode && lines.length === 1 && !encodeOnSeparateLine) {
|
|
882
|
+
lines[lines.length - 1] += ".encode()";
|
|
883
|
+
encode = false;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
return [lines.join("\n") + "\n", encode];
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
export function formatDataAsJson(
|
|
890
|
+
d: DataParam,
|
|
891
|
+
imports: Set<string>,
|
|
892
|
+
osVars: OSVars,
|
|
893
|
+
): [string | null, boolean] {
|
|
894
|
+
if (d instanceof Word) {
|
|
895
|
+
if (!d.isString()) {
|
|
896
|
+
// TODO: a JSON parser that allows shell variables
|
|
897
|
+
return [null, false];
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
// Try to parse using lossless-json first, then fall back to JSON.parse
|
|
901
|
+
// TODO: repeated dictionary keys are discarded
|
|
902
|
+
// https://github.com/josdejong/lossless-json/issues/244
|
|
903
|
+
let dataAsJson;
|
|
904
|
+
try {
|
|
905
|
+
// TODO: types
|
|
906
|
+
// https://github.com/josdejong/lossless-json/issues/245
|
|
907
|
+
dataAsJson = jsonParseLossless(d.toString()) as any;
|
|
908
|
+
} catch {
|
|
909
|
+
try {
|
|
910
|
+
dataAsJson = JSON.parse(d.toString());
|
|
911
|
+
} catch {
|
|
912
|
+
return [null, false];
|
|
913
|
+
}
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
try {
|
|
917
|
+
const jsonDataString = "json_data = " + objToPython(dataAsJson) + "\n";
|
|
918
|
+
// JSON might not be serialized by Python exactly as it was originally
|
|
919
|
+
// due to different whitespace, float formatting like extra + in exponent
|
|
920
|
+
// (1e100 vs 1e+100), different escape sequences in strings
|
|
921
|
+
// ("\/" vs "/" or "\u0008" vs "\b") or duplicate object keys.
|
|
922
|
+
let jsonRoundtrips = false;
|
|
923
|
+
try {
|
|
924
|
+
jsonRoundtrips = jsonDumps(dataAsJson) === d.tokens[0];
|
|
925
|
+
} catch {}
|
|
926
|
+
return [jsonDataString, jsonRoundtrips];
|
|
927
|
+
} catch {}
|
|
928
|
+
} else if (d.filetype === "json") {
|
|
929
|
+
let jsonDataString = "";
|
|
930
|
+
jsonDataString +=
|
|
931
|
+
"with open(" +
|
|
932
|
+
repr(d.filename, osVars, imports, false, true) +
|
|
933
|
+
") as f:\n";
|
|
934
|
+
jsonDataString += " json_data = json.load(f)\n";
|
|
935
|
+
imports.add("json");
|
|
936
|
+
return [jsonDataString, false];
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
return [null, false];
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
function getDataString(
|
|
943
|
+
request: Request,
|
|
944
|
+
osVars: OSVars,
|
|
945
|
+
warnings: Warnings,
|
|
946
|
+
): [string | null, boolean | null, string | null, Set<string>] {
|
|
947
|
+
const imports = new Set<string>();
|
|
948
|
+
if (!request.data || !request.dataArray) {
|
|
949
|
+
return [null, false, null, imports];
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
// There's 4 ways to pass data to Requests (in descending order of preference):
|
|
953
|
+
// a or dictionary/list as the json= argument
|
|
954
|
+
// a dictionary, or a list of tuples (if the dictionary would have duplicate keys) as the data= argument
|
|
955
|
+
// a string as data=
|
|
956
|
+
// bytes as data=
|
|
957
|
+
|
|
958
|
+
// We can pass json= if the data is valid JSON and we've specified json in the
|
|
959
|
+
// Content-Type header because passing json= will set that header.
|
|
960
|
+
//
|
|
961
|
+
// However, if there will be a mismatch between how the JSON is formatted
|
|
962
|
+
// we need to output a commented out version of the request with data= as well.
|
|
963
|
+
// This can happen when there's extra whitespace in the original data or
|
|
964
|
+
// because the JSON contains numbers that are too big to be stored in
|
|
965
|
+
// JavaScript or because there's objects with duplicate keys.
|
|
966
|
+
const contentType = request.headers.get("content-type");
|
|
967
|
+
let dataAsJson: string | null = null;
|
|
968
|
+
let jsonRoundtrips = false;
|
|
969
|
+
if (
|
|
970
|
+
request.dataArray.length === 1 &&
|
|
971
|
+
contentType &&
|
|
972
|
+
contentType.split(";")[0].toString().trim() === "application/json"
|
|
973
|
+
) {
|
|
974
|
+
[dataAsJson, jsonRoundtrips] = formatDataAsJson(
|
|
975
|
+
request.dataArray[0],
|
|
976
|
+
imports,
|
|
977
|
+
osVars,
|
|
978
|
+
);
|
|
979
|
+
}
|
|
980
|
+
if (jsonRoundtrips) {
|
|
981
|
+
return [null, false, dataAsJson, imports];
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
// data= can't be a dict or a list of tuples (i.e. entries) when
|
|
985
|
+
// there is a @file from --data, --data-binary or --json (because they can contain an '&' which would get escaped)
|
|
986
|
+
// there is a --data-urlencode without a name= or name@
|
|
987
|
+
// if you split the input on & and there's a value that doesn't contain an = (e.g. --data "foo=bar&" or simply --data "&")
|
|
988
|
+
// there is a name or value that doesn't roundtrip through percent encoding
|
|
989
|
+
const dataAsEntries = formatDataAsEntries(request.dataArray, osVars, imports);
|
|
990
|
+
if (dataAsEntries !== null) {
|
|
991
|
+
const [dataEntries, percentWarn] = dataAsEntries;
|
|
992
|
+
if (
|
|
993
|
+
eq(
|
|
994
|
+
request.headers.get("content-type"),
|
|
995
|
+
"application/x-www-form-urlencoded",
|
|
996
|
+
) &&
|
|
997
|
+
request.headers.length === 1
|
|
998
|
+
) {
|
|
999
|
+
// Requests adds the header when you include a body
|
|
1000
|
+
// so if it's the only header, don't print the headers
|
|
1001
|
+
request.headers.delete("content-type");
|
|
1002
|
+
}
|
|
1003
|
+
if (percentWarn) {
|
|
1004
|
+
warnings.push([
|
|
1005
|
+
"percent-encoded-spaces-in-data",
|
|
1006
|
+
'data contains spaces encoded by curl as "%20" which will be sent as "+" instead: ' +
|
|
1007
|
+
JSON.stringify(percentWarn),
|
|
1008
|
+
]);
|
|
1009
|
+
}
|
|
1010
|
+
return [dataEntries, false, dataAsJson, imports];
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
const [dataAsString, shouldEncode] = formatDataAsStr(
|
|
1014
|
+
request.dataArray,
|
|
1015
|
+
imports,
|
|
1016
|
+
osVars,
|
|
1017
|
+
);
|
|
1018
|
+
return [dataAsString, shouldEncode, dataAsJson, imports];
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
function getFilesString(
|
|
1022
|
+
request: Request,
|
|
1023
|
+
osVars: OSVars,
|
|
1024
|
+
imports: Set<string>,
|
|
1025
|
+
): [string, boolean] {
|
|
1026
|
+
let usesStdin = false;
|
|
1027
|
+
if (!request.multipartUploads) {
|
|
1028
|
+
return ["", usesStdin];
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
const multipartUploads = request.multipartUploads.map((m) => {
|
|
1032
|
+
// https://github.com/psf/requests/blob/2d5517682b3b38547634d153cea43d48fbc8cdb5/requests/models.py#L117
|
|
1033
|
+
//
|
|
1034
|
+
// Requests's multipart syntax looks like this:
|
|
1035
|
+
// name/filename: content
|
|
1036
|
+
// name: open(filename/contentFile)
|
|
1037
|
+
// name: (filename, open(contentFile)[, contentType[, headers]]))
|
|
1038
|
+
const name = m.name ? repr(m.name, osVars, imports) : "None";
|
|
1039
|
+
|
|
1040
|
+
if (!("contentType" in m) && !("headers" in m) && !("encoder" in m)) {
|
|
1041
|
+
if (
|
|
1042
|
+
"contentFile" in m &&
|
|
1043
|
+
eq(m.contentFile, m.filename) &&
|
|
1044
|
+
!eq(m.contentFile, "-")
|
|
1045
|
+
) {
|
|
1046
|
+
return [
|
|
1047
|
+
name,
|
|
1048
|
+
"open(" +
|
|
1049
|
+
repr(m.contentFile, osVars, imports, false, true) +
|
|
1050
|
+
", 'rb')",
|
|
1051
|
+
];
|
|
1052
|
+
}
|
|
1053
|
+
if ("content" in m && "filename" in m && eq(m.name, m.filename)) {
|
|
1054
|
+
return [name, repr(m.content, osVars, imports)];
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
const sentFilename =
|
|
1059
|
+
"filename" in m && m.filename
|
|
1060
|
+
? repr(m.filename, osVars, imports)
|
|
1061
|
+
: "None";
|
|
1062
|
+
const tuple = [sentFilename];
|
|
1063
|
+
if ("contentFile" in m) {
|
|
1064
|
+
if (eq(m.contentFile, "-")) {
|
|
1065
|
+
// TODO: use piped stdin if we have it
|
|
1066
|
+
usesStdin = true;
|
|
1067
|
+
tuple.push("sys.stdin.buffer.read())");
|
|
1068
|
+
} else {
|
|
1069
|
+
tuple.push(
|
|
1070
|
+
"open(" +
|
|
1071
|
+
repr(m.contentFile, osVars, imports, false, true) +
|
|
1072
|
+
", 'rb')",
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
} else {
|
|
1076
|
+
tuple.push(repr(m.content, osVars, imports));
|
|
1077
|
+
}
|
|
1078
|
+
let addedContentType = false;
|
|
1079
|
+
if ("contentType" in m && m.contentType) {
|
|
1080
|
+
tuple.push(repr(m.contentType, osVars, imports));
|
|
1081
|
+
addedContentType = true;
|
|
1082
|
+
}
|
|
1083
|
+
if (
|
|
1084
|
+
("headers" in m && m.headers) ||
|
|
1085
|
+
("headerFiles" in m && m.headerFiles)
|
|
1086
|
+
) {
|
|
1087
|
+
if (!addedContentType) {
|
|
1088
|
+
tuple.push("None");
|
|
1089
|
+
}
|
|
1090
|
+
|
|
1091
|
+
const headerArg = [];
|
|
1092
|
+
if (m.headers) {
|
|
1093
|
+
let headerDict = "{";
|
|
1094
|
+
for (const header of m.headers) {
|
|
1095
|
+
// TODO: can have multiple headers in one headers= argument?
|
|
1096
|
+
const [name, value] = header.split(":", 2);
|
|
1097
|
+
headerDict +=
|
|
1098
|
+
repr(name, osVars, imports) +
|
|
1099
|
+
": " +
|
|
1100
|
+
repr(value.trimStart() || new Word(), osVars, imports) +
|
|
1101
|
+
", ";
|
|
1102
|
+
}
|
|
1103
|
+
if (headerDict.endsWith(", ")) {
|
|
1104
|
+
headerDict = headerDict.slice(0, -2);
|
|
1105
|
+
}
|
|
1106
|
+
headerDict += "}";
|
|
1107
|
+
headerArg.push(headerDict);
|
|
1108
|
+
}
|
|
1109
|
+
if (m.headerFiles && m.headerFiles.length) {
|
|
1110
|
+
for (const headerFile of m.headerFiles) {
|
|
1111
|
+
headerArg.push(
|
|
1112
|
+
"read_headerfile(" +
|
|
1113
|
+
repr(headerFile, osVars, imports, false, true) +
|
|
1114
|
+
")",
|
|
1115
|
+
);
|
|
1116
|
+
}
|
|
1117
|
+
imports.add("read_headerfile");
|
|
1118
|
+
}
|
|
1119
|
+
tuple.push(headerArg.join(" | "));
|
|
1120
|
+
}
|
|
1121
|
+
return [name, "(" + tuple.join(", ") + ")"];
|
|
1122
|
+
});
|
|
1123
|
+
|
|
1124
|
+
const multipartUploadsAsDict = Object.fromEntries(multipartUploads);
|
|
1125
|
+
|
|
1126
|
+
let filesString = "files = ";
|
|
1127
|
+
if (Object.keys(multipartUploadsAsDict).length === multipartUploads.length) {
|
|
1128
|
+
filesString += "{\n";
|
|
1129
|
+
for (const [multipartKey, multipartValue] of multipartUploads) {
|
|
1130
|
+
filesString += " " + multipartKey + ": " + multipartValue + ",\n";
|
|
1131
|
+
}
|
|
1132
|
+
filesString += "}\n";
|
|
1133
|
+
} else {
|
|
1134
|
+
filesString += "[\n";
|
|
1135
|
+
for (const [multipartKey, multipartValue] of multipartUploads) {
|
|
1136
|
+
filesString += " (" + multipartKey + ", " + multipartValue + "),\n";
|
|
1137
|
+
}
|
|
1138
|
+
filesString += "]\n";
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
return [filesString, usesStdin];
|
|
1142
|
+
}
|
|
1143
|
+
|
|
1144
|
+
// Don't add indent/comment characters to empty lines, most importantly the last line
|
|
1145
|
+
// which will be empty when there's a trailing newline.
|
|
1146
|
+
function indent(s: string, level: number) {
|
|
1147
|
+
if (level === 0) {
|
|
1148
|
+
return s;
|
|
1149
|
+
}
|
|
1150
|
+
const begin = " ".repeat(level);
|
|
1151
|
+
return s
|
|
1152
|
+
.split("\n")
|
|
1153
|
+
.map((l) => (l.trim() ? begin + l : l))
|
|
1154
|
+
.join("\n");
|
|
1155
|
+
}
|
|
1156
|
+
function commentOut(s: string) {
|
|
1157
|
+
return s
|
|
1158
|
+
.split("\n")
|
|
1159
|
+
.map((l) => (l.trim() ? "#" + l : l))
|
|
1160
|
+
.join("\n");
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
function uniqueWarn(
|
|
1164
|
+
seenWarnings: Set<string>,
|
|
1165
|
+
warnings: Warnings,
|
|
1166
|
+
warning: [string, string],
|
|
1167
|
+
) {
|
|
1168
|
+
if (!seenWarnings.has(warning[0])) {
|
|
1169
|
+
seenWarnings.add(warning[0]);
|
|
1170
|
+
warnings.push(warning);
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
function joinArgs(args: string[]) {
|
|
1175
|
+
let s = "(";
|
|
1176
|
+
if (args.join("").length < 100) {
|
|
1177
|
+
s += args.join(", ");
|
|
1178
|
+
} else {
|
|
1179
|
+
s += "\n";
|
|
1180
|
+
for (const arg of args) {
|
|
1181
|
+
s += " " + arg + ",\n";
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
return s + ")";
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
function requestToPython(
|
|
1188
|
+
request: Request,
|
|
1189
|
+
warnings: Warnings = [],
|
|
1190
|
+
imports: Set<string>,
|
|
1191
|
+
thirdPartyImports: Set<string>,
|
|
1192
|
+
): string {
|
|
1193
|
+
const osVars: OSVars = {};
|
|
1194
|
+
const commentedOutHeaders: { [key: string]: string } = {
|
|
1195
|
+
// TODO: add a warning why this should be commented out?
|
|
1196
|
+
"accept-encoding": "",
|
|
1197
|
+
"content-length": "",
|
|
1198
|
+
};
|
|
1199
|
+
// https://github.com/icing/blog/blob/main/curl_on_a_weekend.md
|
|
1200
|
+
if (eq(request.headers.get("te"), "trailers")) {
|
|
1201
|
+
commentedOutHeaders.te = "Requests doesn't support trailers";
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
let cookieStr;
|
|
1205
|
+
let cookieFile: Word | null = null;
|
|
1206
|
+
if (request.cookies) {
|
|
1207
|
+
// TODO: handle duplicate cookie names
|
|
1208
|
+
cookieStr = "cookies = {\n";
|
|
1209
|
+
for (const [cookieName, cookieValue] of request.cookies) {
|
|
1210
|
+
cookieStr +=
|
|
1211
|
+
" " +
|
|
1212
|
+
repr(cookieName, osVars, imports) +
|
|
1213
|
+
": " +
|
|
1214
|
+
repr(cookieValue, osVars, imports) +
|
|
1215
|
+
",\n";
|
|
1216
|
+
}
|
|
1217
|
+
cookieStr += "}\n";
|
|
1218
|
+
// Before Python 3.11, cookies= was sorted alphabetically
|
|
1219
|
+
// https://github.com/python/cpython/issues/86232
|
|
1220
|
+
commentedOutHeaders.cookie = "";
|
|
1221
|
+
if (request.cookieFiles) {
|
|
1222
|
+
warnings.push([
|
|
1223
|
+
"cookie-files",
|
|
1224
|
+
"passing both cookies and cookie files with --cookie/-b is not supported",
|
|
1225
|
+
]);
|
|
1226
|
+
}
|
|
1227
|
+
if (request.cookieJar) {
|
|
1228
|
+
warnings.push([
|
|
1229
|
+
"cookie-files",
|
|
1230
|
+
"passing both cookies and --cookie-jar/-c is not supported",
|
|
1231
|
+
]);
|
|
1232
|
+
}
|
|
1233
|
+
} else if (
|
|
1234
|
+
(request.cookieFiles && request.cookieFiles.length) ||
|
|
1235
|
+
request.cookieJar
|
|
1236
|
+
) {
|
|
1237
|
+
imports.add("http.cookiejar.MozillaCookieJar");
|
|
1238
|
+
if (request.cookieFiles && request.cookieFiles.length) {
|
|
1239
|
+
// TODO: what if user passes multiple cookie files?
|
|
1240
|
+
// TODO: what if user passes cookies and cookie files?
|
|
1241
|
+
cookieFile = request.cookieFiles[request.cookieFiles.length - 1];
|
|
1242
|
+
if (request.cookieFiles.length > 1) {
|
|
1243
|
+
warnings.push([
|
|
1244
|
+
"cookie-files",
|
|
1245
|
+
// TODO: curl reads all of them.
|
|
1246
|
+
"multiple cookie files are not supported, using the last one: " +
|
|
1247
|
+
JSON.stringify(cookieFile.toString()),
|
|
1248
|
+
]);
|
|
1249
|
+
}
|
|
1250
|
+
// TODO: do we need to .load()?
|
|
1251
|
+
cookieStr =
|
|
1252
|
+
"cookies = MozillaCookieJar(" +
|
|
1253
|
+
repr(cookieFile, osVars, imports) +
|
|
1254
|
+
")\n";
|
|
1255
|
+
} else if (request.cookieJar) {
|
|
1256
|
+
cookieStr = "cookies = MozillaCookieJar()\n";
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
let proxyDict;
|
|
1261
|
+
if (request.proxy) {
|
|
1262
|
+
let proxy = request.proxy.includes("://")
|
|
1263
|
+
? request.proxy
|
|
1264
|
+
: request.proxy.prepend("http://");
|
|
1265
|
+
const protocol = proxy.split("://")[0].toLowerCase();
|
|
1266
|
+
if (eq(protocol, "socks")) {
|
|
1267
|
+
// https://github.com/curl/curl/blob/curl-7_86_0/lib/url.c#L2418-L2419
|
|
1268
|
+
proxy = proxy.replace("socks", "socks4");
|
|
1269
|
+
}
|
|
1270
|
+
proxyDict = "proxies = {\n";
|
|
1271
|
+
proxyDict += " 'http': " + repr(proxy, osVars, imports) + ",\n";
|
|
1272
|
+
// TODO: if (protocol !== "http") { ?
|
|
1273
|
+
proxyDict += " 'https': " + repr(proxy, osVars, imports) + ",\n";
|
|
1274
|
+
proxyDict += "}\n";
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
let certStr;
|
|
1278
|
+
if (request.cert || request.key) {
|
|
1279
|
+
certStr = "cert = ";
|
|
1280
|
+
let certPart = "None";
|
|
1281
|
+
if (request.cert) {
|
|
1282
|
+
const [cert, password] = request.cert;
|
|
1283
|
+
certPart = repr(cert, osVars, imports);
|
|
1284
|
+
if (password) {
|
|
1285
|
+
warnings.push([
|
|
1286
|
+
"cert-password",
|
|
1287
|
+
"passing a password to --cert is not supported: " +
|
|
1288
|
+
JSON.stringify(password.toString()),
|
|
1289
|
+
]);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
if (request.key) {
|
|
1293
|
+
certStr +=
|
|
1294
|
+
"(" + certPart + ", " + repr(request.key, osVars, imports) + ")";
|
|
1295
|
+
} else {
|
|
1296
|
+
certStr += certPart;
|
|
1297
|
+
}
|
|
1298
|
+
certStr += "\n";
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
// if there's only 1 URL, put params all together here, unless it's just one string.
|
|
1302
|
+
// if there's more than 1, if we have params that are added to each URL from
|
|
1303
|
+
// --get --data or --url-query that need to read a file, put just the shared part
|
|
1304
|
+
// here, then keep the query in the URL, in the URL.
|
|
1305
|
+
// If there's no --get --data or --url-query, then
|
|
1306
|
+
// put params (if it can be rendered as a list or dict) right before the requests line
|
|
1307
|
+
// Otherwise, keep the query in the URL.
|
|
1308
|
+
let paramsStr;
|
|
1309
|
+
let shouldEncodeParams; // TODO: necessary?
|
|
1310
|
+
function readsFile(paramArray: DataParam[]) {
|
|
1311
|
+
return paramArray.some((p) => !(p instanceof Word));
|
|
1312
|
+
}
|
|
1313
|
+
const paramArray =
|
|
1314
|
+
request.urls.length === 1 ? request.urls[0].queryArray : request.queryArray;
|
|
1315
|
+
if (
|
|
1316
|
+
paramArray &&
|
|
1317
|
+
(request.urls.length === 1 ||
|
|
1318
|
+
(request.urls.length > 1 && readsFile(paramArray)))
|
|
1319
|
+
) {
|
|
1320
|
+
const queryAsEntries = formatDataAsEntries(
|
|
1321
|
+
paramArray,
|
|
1322
|
+
osVars,
|
|
1323
|
+
imports,
|
|
1324
|
+
"params",
|
|
1325
|
+
);
|
|
1326
|
+
if (queryAsEntries !== null) {
|
|
1327
|
+
let percentWarn;
|
|
1328
|
+
[paramsStr, percentWarn] = queryAsEntries;
|
|
1329
|
+
|
|
1330
|
+
if (percentWarn) {
|
|
1331
|
+
warnings.push([
|
|
1332
|
+
"percent-encoded-spaces-in-query",
|
|
1333
|
+
// TODO: will they?
|
|
1334
|
+
'URL querystring contains spaces encoded as "%20" which will be sent by Requests as "+" instead: ' +
|
|
1335
|
+
JSON.stringify(percentWarn),
|
|
1336
|
+
]);
|
|
1337
|
+
}
|
|
1338
|
+
} else if (readsFile(paramArray)) {
|
|
1339
|
+
[paramsStr, shouldEncodeParams] = formatDataAsStr(
|
|
1340
|
+
paramArray,
|
|
1341
|
+
imports,
|
|
1342
|
+
osVars,
|
|
1343
|
+
"params",
|
|
1344
|
+
);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
const contentType = request.headers.get("content-type");
|
|
1349
|
+
let dataString;
|
|
1350
|
+
let jsonDataString;
|
|
1351
|
+
let filesString;
|
|
1352
|
+
let shouldEncode;
|
|
1353
|
+
if (request.urls[0].uploadFile && request.urls.length === 1) {
|
|
1354
|
+
// TODO: https://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
|
|
1355
|
+
if (
|
|
1356
|
+
eq(request.urls[0].uploadFile, "-") ||
|
|
1357
|
+
eq(request.urls[0].uploadFile, ".")
|
|
1358
|
+
) {
|
|
1359
|
+
dataString = "data = sys.stdin.buffer.read()\n";
|
|
1360
|
+
imports.add("sys");
|
|
1361
|
+
} else {
|
|
1362
|
+
dataString =
|
|
1363
|
+
"with open(" +
|
|
1364
|
+
repr(request.urls[0].uploadFile, osVars, imports, false, true) +
|
|
1365
|
+
", 'rb') as f:\n";
|
|
1366
|
+
dataString += " data = f.read()\n";
|
|
1367
|
+
}
|
|
1368
|
+
} else if (request.multipartUploads) {
|
|
1369
|
+
let usesStdin = false;
|
|
1370
|
+
[filesString, usesStdin] = getFilesString(request, osVars, imports);
|
|
1371
|
+
if (usesStdin) {
|
|
1372
|
+
imports.add("sys");
|
|
1373
|
+
}
|
|
1374
|
+
// If you pass files= then Requests adds this header and a `boundary`
|
|
1375
|
+
// If you manually pass a Content-Type header it won't set a `boundary`
|
|
1376
|
+
// wheras curl does, so the request will fail.
|
|
1377
|
+
// https://github.com/curlconverter/curlconverter/issues/248
|
|
1378
|
+
if (
|
|
1379
|
+
filesString &&
|
|
1380
|
+
contentType &&
|
|
1381
|
+
eq(contentType.trim(), "multipart/form-data") &&
|
|
1382
|
+
!contentType.includes("boundary=")
|
|
1383
|
+
) {
|
|
1384
|
+
// TODO: better wording
|
|
1385
|
+
commentedOutHeaders["content-type"] =
|
|
1386
|
+
"requests won't add a boundary if this header is set when you pass files=";
|
|
1387
|
+
}
|
|
1388
|
+
} else if (request.data && !request.data.isEmpty()) {
|
|
1389
|
+
// !isEmpty() because passing data='' is the same as not passing data=
|
|
1390
|
+
// We need to set the Content-Type header in headers= and not set data=
|
|
1391
|
+
let dataImports: Set<string>;
|
|
1392
|
+
[dataString, shouldEncode, jsonDataString, dataImports] = getDataString(
|
|
1393
|
+
request,
|
|
1394
|
+
osVars,
|
|
1395
|
+
warnings,
|
|
1396
|
+
);
|
|
1397
|
+
dataImports.forEach(imports.add, imports);
|
|
1398
|
+
// Remove "Content-Type" from the headers dict
|
|
1399
|
+
// because Requests adds it automatically when you use json=
|
|
1400
|
+
if (
|
|
1401
|
+
jsonDataString &&
|
|
1402
|
+
!dataString &&
|
|
1403
|
+
contentType &&
|
|
1404
|
+
eq(contentType.trim(), "application/json")
|
|
1405
|
+
) {
|
|
1406
|
+
commentedOutHeaders["content-type"] = "Already added when you pass json=";
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
let headerDict;
|
|
1411
|
+
if (request.headers.length) {
|
|
1412
|
+
headerDict = formatHeaders(
|
|
1413
|
+
request.headers,
|
|
1414
|
+
commentedOutHeaders,
|
|
1415
|
+
osVars,
|
|
1416
|
+
imports,
|
|
1417
|
+
);
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
let pythonCode = "";
|
|
1421
|
+
|
|
1422
|
+
if (proxyDict) {
|
|
1423
|
+
pythonCode += proxyDict + "\n";
|
|
1424
|
+
}
|
|
1425
|
+
if (cookieStr) {
|
|
1426
|
+
pythonCode += cookieStr + "\n";
|
|
1427
|
+
}
|
|
1428
|
+
if (headerDict) {
|
|
1429
|
+
pythonCode += headerDict + "\n";
|
|
1430
|
+
}
|
|
1431
|
+
if (paramsStr) {
|
|
1432
|
+
pythonCode += paramsStr + "\n";
|
|
1433
|
+
}
|
|
1434
|
+
if (certStr) {
|
|
1435
|
+
pythonCode += certStr + "\n";
|
|
1436
|
+
}
|
|
1437
|
+
if (jsonDataString) {
|
|
1438
|
+
pythonCode += jsonDataString + "\n";
|
|
1439
|
+
} else if (dataString) {
|
|
1440
|
+
pythonCode += dataString + "\n";
|
|
1441
|
+
} else if (filesString) {
|
|
1442
|
+
pythonCode += filesString + "\n";
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
// By default, curl doesn't follow redirects and Requests does.
|
|
1446
|
+
// Unless redirect behavior has been explicitly set with -L/--location/--no-location
|
|
1447
|
+
// or --max-redirs 0 we pretend generate code that follows redirects,
|
|
1448
|
+
// because adding allow_redirects=False to almost every command would be ugly
|
|
1449
|
+
// and it only matters when the server responds with a redirect, which isn't
|
|
1450
|
+
// that common.
|
|
1451
|
+
let followRedirects = request.followRedirects;
|
|
1452
|
+
let maxRedirects = undefined;
|
|
1453
|
+
if (request.maxRedirects !== undefined) {
|
|
1454
|
+
maxRedirects = asInt(request.maxRedirects, osVars, imports);
|
|
1455
|
+
}
|
|
1456
|
+
if (followRedirects === undefined) {
|
|
1457
|
+
followRedirects = true;
|
|
1458
|
+
|
|
1459
|
+
// Users would see this warning for most commands
|
|
1460
|
+
// warnings.push([
|
|
1461
|
+
// "--location",
|
|
1462
|
+
// "Requests defaults to following redirects, curl doesn't",
|
|
1463
|
+
// ]);
|
|
1464
|
+
}
|
|
1465
|
+
const hasMaxRedirects =
|
|
1466
|
+
followRedirects &&
|
|
1467
|
+
maxRedirects &&
|
|
1468
|
+
maxRedirects !== "0" &&
|
|
1469
|
+
maxRedirects !== "30"; // Requests default
|
|
1470
|
+
|
|
1471
|
+
// Things that vary per-url:
|
|
1472
|
+
// method (because --upload-file can make it PUT)
|
|
1473
|
+
// data= (because of --upload-file)
|
|
1474
|
+
// --output file
|
|
1475
|
+
// params= (because of the query string)
|
|
1476
|
+
// auth= (because the URL can have an auth string)
|
|
1477
|
+
const seenWarnings: Set<string> = new Set();
|
|
1478
|
+
const requestLines = [];
|
|
1479
|
+
let extraEmptyLine = false;
|
|
1480
|
+
for (const [urlObjIndex, urlObj] of request.urls.entries()) {
|
|
1481
|
+
const requestsMethods = [
|
|
1482
|
+
"GET",
|
|
1483
|
+
"HEAD",
|
|
1484
|
+
"POST",
|
|
1485
|
+
"PATCH",
|
|
1486
|
+
"PUT",
|
|
1487
|
+
"DELETE",
|
|
1488
|
+
"OPTIONS", // undocumented
|
|
1489
|
+
];
|
|
1490
|
+
let fn;
|
|
1491
|
+
const args = [];
|
|
1492
|
+
const methodAsString = urlObj.method.toString();
|
|
1493
|
+
if (urlObj.method.isString() && requestsMethods.includes(methodAsString)) {
|
|
1494
|
+
fn = methodAsString.toLowerCase();
|
|
1495
|
+
} else {
|
|
1496
|
+
fn = "request";
|
|
1497
|
+
args.push(repr(urlObj.method, osVars, imports));
|
|
1498
|
+
|
|
1499
|
+
if (
|
|
1500
|
+
urlObj.method.isString() &&
|
|
1501
|
+
methodAsString !== methodAsString.toUpperCase()
|
|
1502
|
+
) {
|
|
1503
|
+
warnings.push([
|
|
1504
|
+
"method",
|
|
1505
|
+
"Requests will uppercase the HTTP method: " +
|
|
1506
|
+
JSON.stringify(methodAsString),
|
|
1507
|
+
]);
|
|
1508
|
+
}
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
let urlParamsStr;
|
|
1512
|
+
let url = urlObj.url;
|
|
1513
|
+
if (request.urls.length === 1) {
|
|
1514
|
+
if (paramsStr) {
|
|
1515
|
+
url = urlObj.urlWithoutQueryArray;
|
|
1516
|
+
} else {
|
|
1517
|
+
url = urlObj.url;
|
|
1518
|
+
}
|
|
1519
|
+
} else {
|
|
1520
|
+
if (paramsStr) {
|
|
1521
|
+
url = urlObj.urlWithOriginalQuery;
|
|
1522
|
+
} else {
|
|
1523
|
+
if (urlObj.queryArray && urlObj.queryArray.length > 0) {
|
|
1524
|
+
const urlQueryAsEntries = formatDataAsEntries(
|
|
1525
|
+
urlObj.queryArray,
|
|
1526
|
+
osVars,
|
|
1527
|
+
imports,
|
|
1528
|
+
"params",
|
|
1529
|
+
);
|
|
1530
|
+
if (urlQueryAsEntries !== null) {
|
|
1531
|
+
let percentWarn;
|
|
1532
|
+
[urlParamsStr, percentWarn] = urlQueryAsEntries;
|
|
1533
|
+
url = urlObj.urlWithoutQueryArray;
|
|
1534
|
+
|
|
1535
|
+
if (percentWarn) {
|
|
1536
|
+
warnings.push([
|
|
1537
|
+
"percent-encoded-spaces-in-query",
|
|
1538
|
+
// TODO: will they?
|
|
1539
|
+
'query contains spaces encoded by curl as "%20" which will be sent as "+" instead: ' +
|
|
1540
|
+
JSON.stringify(percentWarn),
|
|
1541
|
+
]);
|
|
1542
|
+
}
|
|
1543
|
+
} else if (readsFile(urlObj.queryArray)) {
|
|
1544
|
+
[urlParamsStr, shouldEncodeParams] = formatDataAsStr(
|
|
1545
|
+
urlObj.queryArray,
|
|
1546
|
+
imports,
|
|
1547
|
+
osVars,
|
|
1548
|
+
"params",
|
|
1549
|
+
);
|
|
1550
|
+
url = urlObj.urlWithoutQueryArray;
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
// url = urlObj.url
|
|
1554
|
+
}
|
|
1555
|
+
}
|
|
1556
|
+
args.push(repr(url, osVars, imports));
|
|
1557
|
+
|
|
1558
|
+
if (paramsStr || urlParamsStr) {
|
|
1559
|
+
args.push("params=params" + (shouldEncodeParams ? ".encode()" : ""));
|
|
1560
|
+
}
|
|
1561
|
+
if (cookieStr && !request.cookieJar) {
|
|
1562
|
+
args.push("cookies=cookies");
|
|
1563
|
+
}
|
|
1564
|
+
if (headerDict) {
|
|
1565
|
+
args.push("headers=headers");
|
|
1566
|
+
}
|
|
1567
|
+
if (urlObj.uploadFile) {
|
|
1568
|
+
if (request.urls.length > 1) {
|
|
1569
|
+
// If there's more than one URL we could have --data for all
|
|
1570
|
+
// of them and --upload-file for just one of them and we can't
|
|
1571
|
+
// overwrite the `data` variable in that case.
|
|
1572
|
+
args.push("data=file_contents");
|
|
1573
|
+
} else {
|
|
1574
|
+
args.push("data=data");
|
|
1575
|
+
}
|
|
1576
|
+
} else if (filesString) {
|
|
1577
|
+
args.push("files=files");
|
|
1578
|
+
} else if (request.data && !request.data.isEmpty()) {
|
|
1579
|
+
if (jsonDataString) {
|
|
1580
|
+
args.push("json=json_data");
|
|
1581
|
+
} else {
|
|
1582
|
+
args.push("data=data" + (shouldEncode ? ".encode()" : ""));
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
if (proxyDict) {
|
|
1586
|
+
args.push("proxies=proxies");
|
|
1587
|
+
}
|
|
1588
|
+
if (certStr) {
|
|
1589
|
+
args.push("cert=cert");
|
|
1590
|
+
}
|
|
1591
|
+
const certOrPath = request.cacert || request.capath;
|
|
1592
|
+
if (request.insecure) {
|
|
1593
|
+
args.push("verify=False");
|
|
1594
|
+
} else if (certOrPath) {
|
|
1595
|
+
args.push("verify=" + repr(certOrPath, osVars, imports));
|
|
1596
|
+
}
|
|
1597
|
+
// TODO: does this header check apply to all auth methods?
|
|
1598
|
+
if (urlObj.auth && !request.headers.has("Authorization")) {
|
|
1599
|
+
const [user, password] = urlObj.auth;
|
|
1600
|
+
let auth =
|
|
1601
|
+
"(" +
|
|
1602
|
+
repr(user, osVars, imports) +
|
|
1603
|
+
", " +
|
|
1604
|
+
repr(password, osVars, imports) +
|
|
1605
|
+
")";
|
|
1606
|
+
switch (request.authType) {
|
|
1607
|
+
case "basic":
|
|
1608
|
+
break;
|
|
1609
|
+
case "digest":
|
|
1610
|
+
thirdPartyImports.add("requests.auth.HTTPDigestAuth");
|
|
1611
|
+
auth = "HTTPDigestAuth" + auth;
|
|
1612
|
+
break;
|
|
1613
|
+
case "ntlm":
|
|
1614
|
+
case "ntlm-wb":
|
|
1615
|
+
thirdPartyImports.add("requests_ntlm.HttpNtlmAuth");
|
|
1616
|
+
auth = "HttpNtlmAuth" + auth;
|
|
1617
|
+
// TODO: this could stop being true
|
|
1618
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1619
|
+
"ntlm",
|
|
1620
|
+
"requests-ntlm is unmaintained",
|
|
1621
|
+
]);
|
|
1622
|
+
break;
|
|
1623
|
+
case "negotiate":
|
|
1624
|
+
thirdPartyImports.add("requests_gssapi.HTTPSPNEGOAuth");
|
|
1625
|
+
auth = "HTTPSPNEGOAuth(";
|
|
1626
|
+
if (request.delegation) {
|
|
1627
|
+
if (eq(request.delegation, "always")) {
|
|
1628
|
+
auth += "delegate=True";
|
|
1629
|
+
} else if (eq(request.delegation, "none")) {
|
|
1630
|
+
auth += "delegate=False";
|
|
1631
|
+
} else {
|
|
1632
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1633
|
+
"delegation",
|
|
1634
|
+
"--delegation value not supported: " +
|
|
1635
|
+
JSON.stringify(request.delegation.toString()),
|
|
1636
|
+
]);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
auth += ")";
|
|
1640
|
+
// TODO: use requests-kerberos instead?
|
|
1641
|
+
// https://star-history.com/#pythongssapi/requests-gssapi&requests/requests-kerberos
|
|
1642
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1643
|
+
"negotiate",
|
|
1644
|
+
"requests-gssapi is a fork of requests-kerberos",
|
|
1645
|
+
]);
|
|
1646
|
+
break;
|
|
1647
|
+
case "aws-sigv4":
|
|
1648
|
+
thirdPartyImports.add("aws_requests_auth.aws_auth.AWSRequestsAuth");
|
|
1649
|
+
// TODO: move this "auth = " to separate line
|
|
1650
|
+
// TODO: try this.
|
|
1651
|
+
auth =
|
|
1652
|
+
"AWSRequestsAuth(aws_access_key=" +
|
|
1653
|
+
repr(user, osVars, imports) +
|
|
1654
|
+
", aws_secret_access_key=" +
|
|
1655
|
+
repr(password, osVars, imports) +
|
|
1656
|
+
// TODO: parse
|
|
1657
|
+
", aws_host=" +
|
|
1658
|
+
repr(request.awsSigV4 || new Word(), osVars, imports) +
|
|
1659
|
+
", aws_region=" +
|
|
1660
|
+
repr(request.awsSigV4 || new Word(), osVars, imports) +
|
|
1661
|
+
", aws_service=" +
|
|
1662
|
+
repr(request.awsSigV4 || new Word(), osVars, imports) +
|
|
1663
|
+
")";
|
|
1664
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1665
|
+
"--aws-sigv4",
|
|
1666
|
+
"--aws-sigv4 value isn't parsed: " +
|
|
1667
|
+
JSON.stringify(
|
|
1668
|
+
request.awsSigV4 ? request.awsSigV4.toString() : "",
|
|
1669
|
+
),
|
|
1670
|
+
]);
|
|
1671
|
+
break;
|
|
1672
|
+
case "bearer":
|
|
1673
|
+
// Shouldn't happen because hasHeader(Authorization) should be true
|
|
1674
|
+
// TODO: use requests-oauthlib
|
|
1675
|
+
break;
|
|
1676
|
+
}
|
|
1677
|
+
args.push("auth=" + auth);
|
|
1678
|
+
}
|
|
1679
|
+
if (request.timeout || request.connectTimeout) {
|
|
1680
|
+
let connectTimeout = null;
|
|
1681
|
+
if (request.connectTimeout) {
|
|
1682
|
+
connectTimeout = asFloat(request.connectTimeout, osVars, imports);
|
|
1683
|
+
}
|
|
1684
|
+
let timeout = null;
|
|
1685
|
+
if (request.timeout) {
|
|
1686
|
+
if (request.timeout.isString()) {
|
|
1687
|
+
timeout = request.timeout.toString();
|
|
1688
|
+
} else {
|
|
1689
|
+
timeout =
|
|
1690
|
+
"float(" +
|
|
1691
|
+
repr(request.timeout, osVars, imports, false, true) +
|
|
1692
|
+
")";
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
if (timeout && connectTimeout && timeout !== connectTimeout) {
|
|
1697
|
+
args.push("timeout=(" + connectTimeout + ", " + timeout + ")");
|
|
1698
|
+
} else if (timeout) {
|
|
1699
|
+
args.push("timeout=" + timeout);
|
|
1700
|
+
} else if (connectTimeout) {
|
|
1701
|
+
args.push("timeout=(" + connectTimeout + ", None)");
|
|
1702
|
+
}
|
|
1703
|
+
|
|
1704
|
+
if (timeout) {
|
|
1705
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1706
|
+
"--max-time",
|
|
1707
|
+
// https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
|
|
1708
|
+
"unlike --max-time, Requests doesn't have a timeout for the whole request, only for the connect and the read",
|
|
1709
|
+
]);
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
if (!followRedirects || maxRedirects === "0") {
|
|
1714
|
+
args.push("allow_redirects=False");
|
|
1715
|
+
} else if (maxRedirects) {
|
|
1716
|
+
if (maxRedirects === "-1") {
|
|
1717
|
+
imports.add("math");
|
|
1718
|
+
maxRedirects = "math.inf";
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
if (followRedirects && request.followRedirectsTrusted) {
|
|
1722
|
+
uniqueWarn(seenWarnings, warnings, [
|
|
1723
|
+
"--location-trusted",
|
|
1724
|
+
"Requests doesn't have an easy way to disable removing the Authorization: header on redirect",
|
|
1725
|
+
]);
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
let requestLine = "";
|
|
1729
|
+
const isSession = hasMaxRedirects || request.cookieJar;
|
|
1730
|
+
const indentLevel = isSession ? 1 : 0;
|
|
1731
|
+
|
|
1732
|
+
if (isSession && urlObjIndex === 0) {
|
|
1733
|
+
requestLine += "with requests.Session() as session:\n";
|
|
1734
|
+
if (hasMaxRedirects) {
|
|
1735
|
+
requestLine += ` session.max_redirects = ${maxRedirects}\n`;
|
|
1736
|
+
}
|
|
1737
|
+
if (request.cookieJar) {
|
|
1738
|
+
requestLine += ` session.cookies = cookies\n`;
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
if (request.urls.length > 1) {
|
|
1743
|
+
if (urlParamsStr) {
|
|
1744
|
+
requestLine += indent(urlParamsStr, indentLevel);
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
if (urlObj.uploadFile) {
|
|
1748
|
+
let uploadFileLine = "";
|
|
1749
|
+
// TODO: https://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
|
|
1750
|
+
if (eq(urlObj.uploadFile, "-") || eq(urlObj.uploadFile, ".")) {
|
|
1751
|
+
uploadFileLine += "file_contents = sys.stdin.buffer.read()\n";
|
|
1752
|
+
imports.add("sys");
|
|
1753
|
+
} else {
|
|
1754
|
+
uploadFileLine +=
|
|
1755
|
+
"with open(" +
|
|
1756
|
+
repr(urlObj.uploadFile, osVars, imports, false, true) +
|
|
1757
|
+
", 'rb') as f:\n";
|
|
1758
|
+
uploadFileLine += " file_contents = f.read()\n";
|
|
1759
|
+
}
|
|
1760
|
+
requestLine += indent(uploadFileLine, indentLevel);
|
|
1761
|
+
}
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
const fnToCall =
|
|
1765
|
+
"response = " + (isSession ? "session" : "requests") + "." + fn;
|
|
1766
|
+
requestLine += indent(fnToCall + joinArgs(args) + "\n", indentLevel);
|
|
1767
|
+
|
|
1768
|
+
if (jsonDataString && dataString && !urlObj.uploadFile) {
|
|
1769
|
+
// Adding empty lines to a "with" block breaks the code when pasted in the REPL
|
|
1770
|
+
requestLine += isSession || request.urls.length > 1 ? "" : "\n";
|
|
1771
|
+
|
|
1772
|
+
// Should never be -1
|
|
1773
|
+
args[args.indexOf("json=json_data")] = shouldEncode
|
|
1774
|
+
? "data=data.encode()"
|
|
1775
|
+
: "data=data";
|
|
1776
|
+
let dataAlternative =
|
|
1777
|
+
"# Note: json_data will not be serialized by requests\n" +
|
|
1778
|
+
"# exactly as it was in the original request.\n";
|
|
1779
|
+
dataAlternative += commentOut(dataString);
|
|
1780
|
+
dataAlternative += commentOut(fnToCall + joinArgs(args) + "\n");
|
|
1781
|
+
requestLine += indent(dataAlternative, indentLevel);
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
if (urlObj.output && !eq(urlObj.output, "/dev/null")) {
|
|
1785
|
+
let outputLine = "";
|
|
1786
|
+
if (eq(request.urls[0].output, "-")) {
|
|
1787
|
+
outputLine += "print(response.text)\n";
|
|
1788
|
+
} else {
|
|
1789
|
+
outputLine += isSession || request.urls.length > 1 ? "" : "\n";
|
|
1790
|
+
|
|
1791
|
+
outputLine +=
|
|
1792
|
+
"with open(" +
|
|
1793
|
+
repr(urlObj.output, osVars, imports, false, true) +
|
|
1794
|
+
", 'wb') as f:\n";
|
|
1795
|
+
outputLine += " f.write(response.content)\n";
|
|
1796
|
+
}
|
|
1797
|
+
requestLine += indent(outputLine, indentLevel);
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
if (
|
|
1801
|
+
!isSession &&
|
|
1802
|
+
// request.urls.length > 1 &&
|
|
1803
|
+
(urlParamsStr ||
|
|
1804
|
+
(dataString && jsonDataString) ||
|
|
1805
|
+
urlObj.uploadFile ||
|
|
1806
|
+
(urlObj.output && !eq(urlObj.output, "/dev/null")))
|
|
1807
|
+
) {
|
|
1808
|
+
extraEmptyLine = true;
|
|
1809
|
+
}
|
|
1810
|
+
requestLines.push(requestLine);
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
pythonCode += requestLines.join(extraEmptyLine ? "\n" : "");
|
|
1814
|
+
|
|
1815
|
+
if (request.cookieJar) {
|
|
1816
|
+
let cookieSaveLine = "cookies.save(";
|
|
1817
|
+
if (!eq(request.cookieJar, cookieFile)) {
|
|
1818
|
+
cookieSaveLine += repr(request.cookieJar, osVars, imports) + ", ";
|
|
1819
|
+
}
|
|
1820
|
+
cookieSaveLine += "ignore_discard=True, ignore_expires=True)\n"; // TODO: necessary?
|
|
1821
|
+
pythonCode += indent(cookieSaveLine, 1);
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
if (request.http3) {
|
|
1825
|
+
warnings.push([
|
|
1826
|
+
"http3",
|
|
1827
|
+
"this was an HTTP/3 request but requests only supports HTTP/1.1",
|
|
1828
|
+
]);
|
|
1829
|
+
} else if (request.http2) {
|
|
1830
|
+
warnings.push([
|
|
1831
|
+
"http2",
|
|
1832
|
+
"this was an HTTP/2 request but requests only supports HTTP/1.1",
|
|
1833
|
+
]);
|
|
1834
|
+
} else if (request.headers.lowercase && request.headers.length > 1) {
|
|
1835
|
+
warnings.push([
|
|
1836
|
+
"--header",
|
|
1837
|
+
"all the --header/-H names are lowercase, which means this may have been an HTTP/2 or HTTP/3 request. Requests only sends HTTP/1.1",
|
|
1838
|
+
]);
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
// It would be more correct to run the commands whenever we need
|
|
1842
|
+
// their values instead of running them all at the beginning.
|
|
1843
|
+
let variableCode = "";
|
|
1844
|
+
if (Object.keys(osVars).length) {
|
|
1845
|
+
for (const [varName, expr] of Object.entries(osVars)) {
|
|
1846
|
+
variableCode += `${varName} = ${expr}\n`;
|
|
1847
|
+
}
|
|
1848
|
+
variableCode += "\n";
|
|
1849
|
+
}
|
|
1850
|
+
|
|
1851
|
+
return variableCode + pythonCode;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
export function printImports(imps: Set<string>): string {
|
|
1855
|
+
let s = "";
|
|
1856
|
+
for (const imp of Array.from(imps).sort()) {
|
|
1857
|
+
if (imp === "read_headerfile") {
|
|
1858
|
+
continue;
|
|
1859
|
+
}
|
|
1860
|
+
if (imp.includes(".")) {
|
|
1861
|
+
const pos = imp.lastIndexOf(".");
|
|
1862
|
+
const module = imp.slice(0, pos);
|
|
1863
|
+
const name = imp.slice(pos + 1);
|
|
1864
|
+
s += "from " + module + " import " + name + "\n";
|
|
1865
|
+
} else {
|
|
1866
|
+
s += "import " + imp + "\n";
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
return s;
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
export function _toPython(
|
|
1873
|
+
requests: Request[],
|
|
1874
|
+
warnings: Warnings = [],
|
|
1875
|
+
): string {
|
|
1876
|
+
const code = [];
|
|
1877
|
+
let joinTwoLines = false;
|
|
1878
|
+
const imports = new Set<string>();
|
|
1879
|
+
const thirdPartyImports = new Set<string>();
|
|
1880
|
+
for (const request of requests) {
|
|
1881
|
+
const requestCode = requestToPython(
|
|
1882
|
+
request,
|
|
1883
|
+
warnings,
|
|
1884
|
+
imports,
|
|
1885
|
+
thirdPartyImports,
|
|
1886
|
+
);
|
|
1887
|
+
code.push(requestCode);
|
|
1888
|
+
|
|
1889
|
+
// If one of the requests defines variables (or its URL is very long),
|
|
1890
|
+
// separate all the configs with two empty lines.
|
|
1891
|
+
// curl example.com --next example.com
|
|
1892
|
+
// vs.
|
|
1893
|
+
// curl --data "foo=bar" example.com --next example.com
|
|
1894
|
+
// (+1 for the trailing newline)
|
|
1895
|
+
joinTwoLines ||= requestCode.split("\n").length > request.urls.length + 1;
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
let importCode = "";
|
|
1899
|
+
importCode += printImports(imports);
|
|
1900
|
+
if (imports.size > 1) {
|
|
1901
|
+
importCode += "\n";
|
|
1902
|
+
}
|
|
1903
|
+
importCode += "import requests\n";
|
|
1904
|
+
importCode += printImports(thirdPartyImports);
|
|
1905
|
+
importCode += "\n";
|
|
1906
|
+
if (imports.has("read_headerfile")) {
|
|
1907
|
+
importCode += "def read_headerfile(filename):\n";
|
|
1908
|
+
importCode += " headers = {}\n";
|
|
1909
|
+
importCode += " with open(filename) as f:\n";
|
|
1910
|
+
importCode += " for line in f:\n";
|
|
1911
|
+
importCode += " line = line.split('#', 1)[0].strip()\n";
|
|
1912
|
+
importCode += " if line:\n";
|
|
1913
|
+
importCode += " [key, value] = line.split(':', 1)\n";
|
|
1914
|
+
importCode += " value = value or ''\n";
|
|
1915
|
+
importCode += " headers[key.strip()] = value.strip()\n";
|
|
1916
|
+
importCode += " return headers\n";
|
|
1917
|
+
importCode += "\n";
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
return importCode + code.join(joinTwoLines ? "\n\n" : "\n");
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
export function toPythonWarn(
|
|
1924
|
+
curlCommand: string | string[],
|
|
1925
|
+
warnings: Warnings = [],
|
|
1926
|
+
): [string, Warnings] {
|
|
1927
|
+
const requests = parse(curlCommand, supportedArgs, warnings);
|
|
1928
|
+
const python = _toPython(requests, warnings);
|
|
1929
|
+
return [python, warnings];
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
export function toPython(curlCommand: string | string[]): string {
|
|
1933
|
+
return toPythonWarn(curlCommand)[0];
|
|
1934
|
+
}
|