@ohos-ports/curlconverter 4.12.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (270) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +153 -0
  3. package/dist/src/Headers.d.ts +23 -0
  4. package/dist/src/Headers.js +287 -0
  5. package/dist/src/Headers.js.map +1 -0
  6. package/dist/src/Query.d.ts +8 -0
  7. package/dist/src/Query.js +136 -0
  8. package/dist/src/Query.js.map +1 -0
  9. package/dist/src/Request.d.ts +200 -0
  10. package/dist/src/Request.js +1191 -0
  11. package/dist/src/Request.js.map +1 -0
  12. package/dist/src/Warnings.d.ts +15 -0
  13. package/dist/src/Warnings.js +111 -0
  14. package/dist/src/Warnings.js.map +1 -0
  15. package/dist/src/cli.d.ts +2 -0
  16. package/dist/src/cli.js +333 -0
  17. package/dist/src/cli.js.map +1 -0
  18. package/dist/src/curl/auth.d.ts +12 -0
  19. package/dist/src/curl/auth.js +40 -0
  20. package/dist/src/curl/auth.js.map +1 -0
  21. package/dist/src/curl/form.d.ts +28 -0
  22. package/dist/src/curl/form.js +218 -0
  23. package/dist/src/curl/form.js.map +1 -0
  24. package/dist/src/curl/opts.d.ts +2052 -0
  25. package/dist/src/curl/opts.js +1000 -0
  26. package/dist/src/curl/opts.js.map +1 -0
  27. package/dist/src/curl/url.d.ts +14 -0
  28. package/dist/src/curl/url.js +218 -0
  29. package/dist/src/curl/url.js.map +1 -0
  30. package/dist/src/generators/ansible.d.ts +5 -0
  31. package/dist/src/generators/ansible.js +282 -0
  32. package/dist/src/generators/ansible.js.map +1 -0
  33. package/dist/src/generators/c.d.ts +12 -0
  34. package/dist/src/generators/c.js +1230 -0
  35. package/dist/src/generators/c.js.map +1 -0
  36. package/dist/src/generators/cfml.d.ts +5 -0
  37. package/dist/src/generators/cfml.js +134 -0
  38. package/dist/src/generators/cfml.js.map +1 -0
  39. package/dist/src/generators/clojure.d.ts +7 -0
  40. package/dist/src/generators/clojure.js +436 -0
  41. package/dist/src/generators/clojure.js.map +1 -0
  42. package/dist/src/generators/csharp.d.ts +8 -0
  43. package/dist/src/generators/csharp.js +356 -0
  44. package/dist/src/generators/csharp.js.map +1 -0
  45. package/dist/src/generators/dart.d.ts +5 -0
  46. package/dist/src/generators/dart.js +273 -0
  47. package/dist/src/generators/dart.js.map +1 -0
  48. package/dist/src/generators/elixir.d.ts +8 -0
  49. package/dist/src/generators/elixir.js +296 -0
  50. package/dist/src/generators/elixir.js.map +1 -0
  51. package/dist/src/generators/go.d.ts +5 -0
  52. package/dist/src/generators/go.js +228 -0
  53. package/dist/src/generators/go.js.map +1 -0
  54. package/dist/src/generators/har.d.ts +7 -0
  55. package/dist/src/generators/har.js +115 -0
  56. package/dist/src/generators/har.js.map +1 -0
  57. package/dist/src/generators/http.d.ts +7 -0
  58. package/dist/src/generators/http.js +165 -0
  59. package/dist/src/generators/http.js.map +1 -0
  60. package/dist/src/generators/httpie.d.ts +5 -0
  61. package/dist/src/generators/httpie.js +446 -0
  62. package/dist/src/generators/httpie.js.map +1 -0
  63. package/dist/src/generators/java/httpurlconnection.d.ts +5 -0
  64. package/dist/src/generators/java/httpurlconnection.js +123 -0
  65. package/dist/src/generators/java/httpurlconnection.js.map +1 -0
  66. package/dist/src/generators/java/java.d.ts +8 -0
  67. package/dist/src/generators/java/java.js +247 -0
  68. package/dist/src/generators/java/java.js.map +1 -0
  69. package/dist/src/generators/java/jsoup.d.ts +5 -0
  70. package/dist/src/generators/java/jsoup.js +152 -0
  71. package/dist/src/generators/java/jsoup.js.map +1 -0
  72. package/dist/src/generators/java/okhttp.d.ts +5 -0
  73. package/dist/src/generators/java/okhttp.js +232 -0
  74. package/dist/src/generators/java/okhttp.js.map +1 -0
  75. package/dist/src/generators/javascript/axios.d.ts +5 -0
  76. package/dist/src/generators/javascript/axios.js +332 -0
  77. package/dist/src/generators/javascript/axios.js.map +1 -0
  78. package/dist/src/generators/javascript/got.d.ts +5 -0
  79. package/dist/src/generators/javascript/got.js +277 -0
  80. package/dist/src/generators/javascript/got.js.map +1 -0
  81. package/dist/src/generators/javascript/http.d.ts +8 -0
  82. package/dist/src/generators/javascript/http.js +183 -0
  83. package/dist/src/generators/javascript/http.js.map +1 -0
  84. package/dist/src/generators/javascript/javascript.d.ts +37 -0
  85. package/dist/src/generators/javascript/javascript.js +676 -0
  86. package/dist/src/generators/javascript/javascript.js.map +1 -0
  87. package/dist/src/generators/javascript/jquery.d.ts +15 -0
  88. package/dist/src/generators/javascript/jquery.js +297 -0
  89. package/dist/src/generators/javascript/jquery.js.map +1 -0
  90. package/dist/src/generators/javascript/ky.d.ts +7 -0
  91. package/dist/src/generators/javascript/ky.js +245 -0
  92. package/dist/src/generators/javascript/ky.js.map +1 -0
  93. package/dist/src/generators/javascript/request.d.ts +5 -0
  94. package/dist/src/generators/javascript/request.js +113 -0
  95. package/dist/src/generators/javascript/request.js.map +1 -0
  96. package/dist/src/generators/javascript/superagent.d.ts +10 -0
  97. package/dist/src/generators/javascript/superagent.js +301 -0
  98. package/dist/src/generators/javascript/superagent.js.map +1 -0
  99. package/dist/src/generators/javascript/xhr.d.ts +8 -0
  100. package/dist/src/generators/javascript/xhr.js +147 -0
  101. package/dist/src/generators/javascript/xhr.js.map +1 -0
  102. package/dist/src/generators/json.d.ts +45 -0
  103. package/dist/src/generators/json.js +197 -0
  104. package/dist/src/generators/json.js.map +1 -0
  105. package/dist/src/generators/julia.d.ts +8 -0
  106. package/dist/src/generators/julia.js +365 -0
  107. package/dist/src/generators/julia.js.map +1 -0
  108. package/dist/src/generators/kotlin.d.ts +8 -0
  109. package/dist/src/generators/kotlin.js +325 -0
  110. package/dist/src/generators/kotlin.js.map +1 -0
  111. package/dist/src/generators/lua.d.ts +5 -0
  112. package/dist/src/generators/lua.js +192 -0
  113. package/dist/src/generators/lua.js.map +1 -0
  114. package/dist/src/generators/matlab/common.d.ts +16 -0
  115. package/dist/src/generators/matlab/common.js +238 -0
  116. package/dist/src/generators/matlab/common.js.map +1 -0
  117. package/dist/src/generators/matlab/httpinterface.d.ts +2 -0
  118. package/dist/src/generators/matlab/httpinterface.js +211 -0
  119. package/dist/src/generators/matlab/httpinterface.js.map +1 -0
  120. package/dist/src/generators/matlab/matlab.d.ts +5 -0
  121. package/dist/src/generators/matlab/matlab.js +29 -0
  122. package/dist/src/generators/matlab/matlab.js.map +1 -0
  123. package/dist/src/generators/matlab/webservices.d.ts +2 -0
  124. package/dist/src/generators/matlab/webservices.js +191 -0
  125. package/dist/src/generators/matlab/webservices.js.map +1 -0
  126. package/dist/src/generators/objectivec.d.ts +8 -0
  127. package/dist/src/generators/objectivec.js +269 -0
  128. package/dist/src/generators/objectivec.js.map +1 -0
  129. package/dist/src/generators/ocaml.d.ts +8 -0
  130. package/dist/src/generators/ocaml.js +150 -0
  131. package/dist/src/generators/ocaml.js.map +1 -0
  132. package/dist/src/generators/perl.d.ts +9 -0
  133. package/dist/src/generators/perl.js +209 -0
  134. package/dist/src/generators/perl.js.map +1 -0
  135. package/dist/src/generators/php/guzzle.d.ts +5 -0
  136. package/dist/src/generators/php/guzzle.js +328 -0
  137. package/dist/src/generators/php/guzzle.js.map +1 -0
  138. package/dist/src/generators/php/php.d.ts +8 -0
  139. package/dist/src/generators/php/php.js +217 -0
  140. package/dist/src/generators/php/php.js.map +1 -0
  141. package/dist/src/generators/php/requests.d.ts +5 -0
  142. package/dist/src/generators/php/requests.js +89 -0
  143. package/dist/src/generators/php/requests.js.map +1 -0
  144. package/dist/src/generators/powershell.d.ts +11 -0
  145. package/dist/src/generators/powershell.js +377 -0
  146. package/dist/src/generators/powershell.js.map +1 -0
  147. package/dist/src/generators/python/http.d.ts +5 -0
  148. package/dist/src/generators/python/http.js +128 -0
  149. package/dist/src/generators/python/http.js.map +1 -0
  150. package/dist/src/generators/python/python.d.ts +22 -0
  151. package/dist/src/generators/python/python.js +1693 -0
  152. package/dist/src/generators/python/python.js.map +1 -0
  153. package/dist/src/generators/r/httr.d.ts +10 -0
  154. package/dist/src/generators/r/httr.js +298 -0
  155. package/dist/src/generators/r/httr.js.map +1 -0
  156. package/dist/src/generators/r/httr2.d.ts +5 -0
  157. package/dist/src/generators/r/httr2.js +214 -0
  158. package/dist/src/generators/r/httr2.js.map +1 -0
  159. package/dist/src/generators/ruby/httparty.d.ts +6 -0
  160. package/dist/src/generators/ruby/httparty.js +308 -0
  161. package/dist/src/generators/ruby/httparty.js.map +1 -0
  162. package/dist/src/generators/ruby/ruby.d.ts +14 -0
  163. package/dist/src/generators/ruby/ruby.js +515 -0
  164. package/dist/src/generators/ruby/ruby.js.map +1 -0
  165. package/dist/src/generators/rust.d.ts +8 -0
  166. package/dist/src/generators/rust.js +177 -0
  167. package/dist/src/generators/rust.js.map +1 -0
  168. package/dist/src/generators/swift.d.ts +8 -0
  169. package/dist/src/generators/swift.js +219 -0
  170. package/dist/src/generators/swift.js.map +1 -0
  171. package/dist/src/generators/wget.d.ts +8 -0
  172. package/dist/src/generators/wget.js +472 -0
  173. package/dist/src/generators/wget.js.map +1 -0
  174. package/dist/src/index.d.ts +53 -0
  175. package/dist/src/index.js +54 -0
  176. package/dist/src/index.js.map +1 -0
  177. package/dist/src/parse.d.ts +13 -0
  178. package/dist/src/parse.js +43 -0
  179. package/dist/src/parse.js.map +1 -0
  180. package/dist/src/shell/Parser.d.ts +4 -0
  181. package/dist/src/shell/Parser.js +6 -0
  182. package/dist/src/shell/Parser.js.map +1 -0
  183. package/dist/src/shell/Word.d.ts +48 -0
  184. package/dist/src/shell/Word.js +486 -0
  185. package/dist/src/shell/Word.js.map +1 -0
  186. package/dist/src/shell/tokenizer.d.ts +3 -0
  187. package/dist/src/shell/tokenizer.js +610 -0
  188. package/dist/src/shell/tokenizer.js.map +1 -0
  189. package/dist/src/shell/webParser.d.ts +4 -0
  190. package/dist/src/shell/webParser.js +15 -0
  191. package/dist/src/shell/webParser.js.map +1 -0
  192. package/dist/src/utils.d.ts +6 -0
  193. package/dist/src/utils.js +21 -0
  194. package/dist/src/utils.js.map +1 -0
  195. package/dist/tools/compare-requests.d.ts +2 -0
  196. package/dist/tools/compare-requests.js +448 -0
  197. package/dist/tools/compare-requests.js.map +1 -0
  198. package/dist/tools/gen-test.d.ts +2 -0
  199. package/dist/tools/gen-test.js +114 -0
  200. package/dist/tools/gen-test.js.map +1 -0
  201. package/dist/tree-sitter-bash.wasm +0 -0
  202. package/index.d.ts +3 -0
  203. package/package.json +97 -0
  204. package/src/Headers.ts +334 -0
  205. package/src/Query.ts +147 -0
  206. package/src/Request.ts +1649 -0
  207. package/src/Warnings.ts +154 -0
  208. package/src/cli.ts +561 -0
  209. package/src/curl/auth.ts +45 -0
  210. package/src/curl/form.ts +297 -0
  211. package/src/curl/opts.ts +1417 -0
  212. package/src/curl/url.ts +241 -0
  213. package/src/generators/ansible.ts +364 -0
  214. package/src/generators/c.ts +1371 -0
  215. package/src/generators/cfml.ts +155 -0
  216. package/src/generators/clojure.ts +521 -0
  217. package/src/generators/csharp.ts +383 -0
  218. package/src/generators/dart.ts +293 -0
  219. package/src/generators/elixir.ts +349 -0
  220. package/src/generators/go.ts +269 -0
  221. package/src/generators/har.ts +147 -0
  222. package/src/generators/http.ts +194 -0
  223. package/src/generators/httpie.ts +506 -0
  224. package/src/generators/java/httpurlconnection.ts +151 -0
  225. package/src/generators/java/java.ts +280 -0
  226. package/src/generators/java/jsoup.ts +172 -0
  227. package/src/generators/java/okhttp.ts +257 -0
  228. package/src/generators/javascript/axios.ts +395 -0
  229. package/src/generators/javascript/got.ts +362 -0
  230. package/src/generators/javascript/http.ts +232 -0
  231. package/src/generators/javascript/javascript.ts +825 -0
  232. package/src/generators/javascript/jquery.ts +360 -0
  233. package/src/generators/javascript/ky.ts +315 -0
  234. package/src/generators/javascript/request.ts +152 -0
  235. package/src/generators/javascript/superagent.ts +373 -0
  236. package/src/generators/javascript/xhr.ts +192 -0
  237. package/src/generators/json.ts +293 -0
  238. package/src/generators/julia.ts +398 -0
  239. package/src/generators/kotlin.ts +355 -0
  240. package/src/generators/lua.ts +209 -0
  241. package/src/generators/matlab/common.ts +281 -0
  242. package/src/generators/matlab/httpinterface.ts +284 -0
  243. package/src/generators/matlab/matlab.ts +39 -0
  244. package/src/generators/matlab/webservices.ts +263 -0
  245. package/src/generators/objectivec.ts +297 -0
  246. package/src/generators/ocaml.ts +168 -0
  247. package/src/generators/perl.ts +237 -0
  248. package/src/generators/php/guzzle.ts +368 -0
  249. package/src/generators/php/php.ts +239 -0
  250. package/src/generators/php/requests.ts +103 -0
  251. package/src/generators/powershell.ts +430 -0
  252. package/src/generators/python/http.ts +167 -0
  253. package/src/generators/python/python.ts +1934 -0
  254. package/src/generators/r/httr.ts +328 -0
  255. package/src/generators/r/httr2.ts +268 -0
  256. package/src/generators/ruby/httparty.ts +351 -0
  257. package/src/generators/ruby/ruby.ts +564 -0
  258. package/src/generators/rust.ts +243 -0
  259. package/src/generators/swift.ts +235 -0
  260. package/src/generators/wget.ts +533 -0
  261. package/src/index.ts +89 -0
  262. package/src/parse.ts +72 -0
  263. package/src/shell/Parser.ts +8 -0
  264. package/src/shell/Word.ts +531 -0
  265. package/src/shell/tokenizer.ts +736 -0
  266. package/src/shell/webParser.ts +18 -0
  267. package/src/utils.ts +25 -0
  268. package/tools/compare-requests.ts +571 -0
  269. package/tools/extract_curl_args.py +511 -0
  270. package/tools/gen-test.ts +129 -0
@@ -0,0 +1,1934 @@
1
+ import { CCError, has, UTF8encoder } from "../../utils.js";
2
+ import { Word, eq } from "../../shell/Word.js";
3
+ import { parse, COMMON_SUPPORTED_ARGS } from "../../parse.js";
4
+ import type { Request, Warnings } from "../../parse.js";
5
+ import { Headers } from "../../Headers.js";
6
+ import { wordDecodeURIComponent, percentEncode } from "../../Query.js";
7
+ import { DataParam } from "../../Request.js";
8
+
9
+ import {
10
+ parse as jsonParseLossless,
11
+ stringify as jsonStringifyLossless,
12
+ isSafeNumber,
13
+ isInteger,
14
+ isLosslessNumber,
15
+ } from "lossless-json";
16
+
17
+ // TODO: partiallySupportedArgs
18
+ export const supportedArgs = new Set([
19
+ ...COMMON_SUPPORTED_ARGS,
20
+
21
+ "compressed",
22
+ // "no-compressed",
23
+
24
+ // "anyauth",
25
+ // "no-anyauth",
26
+ "digest",
27
+ "no-digest",
28
+ "aws-sigv4",
29
+ "negotiate",
30
+ "no-negotiate",
31
+ "delegation", // GSS/kerberos
32
+ // "service-name", // GSS/kerberos, not supported
33
+ "ntlm",
34
+ "no-ntlm",
35
+ "ntlm-wb",
36
+ "no-ntlm-wb",
37
+
38
+ "http1.1",
39
+ // These are not supported, just better warning message
40
+ "http2",
41
+ "http2-prior-knowledge",
42
+ "http3",
43
+ "http3-only",
44
+
45
+ "cookie-jar",
46
+
47
+ "cert",
48
+ "cacert",
49
+ "key",
50
+ "capath",
51
+
52
+ "form",
53
+ "form-string",
54
+
55
+ "location",
56
+ "no-location",
57
+ "location-trusted", // not exactly supported, just better warning message
58
+ "no-location-trusted",
59
+ "max-redirs",
60
+
61
+ "max-time",
62
+ "connect-timeout",
63
+
64
+ "insecure",
65
+ "no-insecure",
66
+
67
+ "output",
68
+ "upload-file",
69
+
70
+ "next",
71
+
72
+ "proxy",
73
+ "proxy-user",
74
+ ]);
75
+
76
+ // https://peps.python.org/pep-3138/
77
+ // https://www.unicode.org/reports/tr44/#GC_Values_Table
78
+ // https://unicode.org/Public/UNIDATA/UnicodeData.txt
79
+ // https://en.wikipedia.org/wiki/Plane_(Unicode)#Overview
80
+ const regexSingleEscape = /'|\\|\p{C}|[^ \P{Z}]/gu;
81
+ const regexDoubleEscape = /"|\\|\p{C}|[^ \P{Z}]/gu;
82
+
83
+ // Also used for Go and R
84
+ export function reprStr(s: string, quote?: '"' | "'"): string {
85
+ if (quote === undefined) {
86
+ quote = "'";
87
+ if (s.includes("'") && !s.includes('"')) {
88
+ quote = '"';
89
+ }
90
+ }
91
+ const regex = quote === "'" ? regexSingleEscape : regexDoubleEscape;
92
+
93
+ return (
94
+ quote +
95
+ s.replace(regex, (c: string): string => {
96
+ switch (c) {
97
+ case "\x07":
98
+ return "\\a";
99
+ case "\b":
100
+ return "\\b";
101
+ case "\f":
102
+ return "\\f";
103
+ case "\n":
104
+ return "\\n";
105
+ case "\r":
106
+ return "\\r";
107
+ case "\t":
108
+ return "\\t";
109
+ case "\v":
110
+ return "\\v";
111
+ case "\\":
112
+ return "\\\\";
113
+ case "'":
114
+ return "\\'";
115
+ case '"':
116
+ return '\\"';
117
+ }
118
+ const hex = (c.codePointAt(0) as number).toString(16);
119
+ if (hex.length <= 2) {
120
+ return "\\x" + hex.padStart(2, "0");
121
+ }
122
+ if (hex.length <= 4) {
123
+ return "\\u" + hex.padStart(4, "0");
124
+ }
125
+ return "\\U" + hex.padStart(8, "0");
126
+ }) +
127
+ quote
128
+ );
129
+ }
130
+
131
+ // TODO: use this if string contains unmatched surrogates?
132
+ // It just replaces them with the replacement character, but at least that code would run.
133
+ export function pybescComplex(s: string): string {
134
+ let quote = "'";
135
+ if (s.includes("'") && !s.includes('"')) {
136
+ quote = '"';
137
+ }
138
+ const quoteCode = quote.charCodeAt(0);
139
+
140
+ // TODO: using UTF-8 here is overly simplistic and how to encode here
141
+ // is a pretty complicated decision.
142
+ // For starters, it would be more correct to use the same encoding as
143
+ // the terminal when running from the command line.
144
+ const bytes = UTF8encoder.encode(s);
145
+
146
+ return (
147
+ "b" +
148
+ quote +
149
+ [...bytes]
150
+ .map((c: number): string => {
151
+ switch (c) {
152
+ case 0x07:
153
+ return "\\a";
154
+ case 0x08:
155
+ return "\\b";
156
+ case 0x0c:
157
+ return "\\f";
158
+ case 0x0a:
159
+ return "\\n";
160
+ case 0x0d:
161
+ return "\\r";
162
+ case 0x09:
163
+ return "\\t";
164
+ case 0x0b:
165
+ return "\\v";
166
+ case 0x5c:
167
+ return "\\\\";
168
+ case quoteCode:
169
+ return "\\" + String.fromCharCode(c);
170
+ }
171
+ if (c >= 0x20 && c < 0x7f) {
172
+ return String.fromCharCode(c);
173
+ }
174
+ const hex = c.toString(16);
175
+ return "\\x" + hex.padStart(2, "0");
176
+ })
177
+ .join("") +
178
+ quote
179
+ );
180
+ }
181
+
182
+ export function reprStrBinary(s: string): string {
183
+ const sEsc = reprStr(s);
184
+ // We check until 0x7F instead of 0xFF because curl (running in a UTF-8 terminal) when it gets
185
+ // bytes sends them as is, but if we pass b'\x80' to Requests, it will encode that byte as
186
+ // Latin-1 (presumably for backwards compatibility) instead of UTF-8.
187
+ if (/^[\x00-\x7f]*$/.test(s)) {
188
+ return "b" + sEsc;
189
+ }
190
+ // TODO: unmatched surrogates will generate code that throws an error
191
+ // e.g.: '\uDC00'.encode()
192
+ return sEsc + ".encode()";
193
+ }
194
+
195
+ export type OSVars = { [key: string]: string };
196
+ export function repr(
197
+ word: Word,
198
+ osVars: OSVars,
199
+ imports: Set<string>,
200
+ binary = false,
201
+ // os.getenv('MYVAR') returns None if MYVAR is not set
202
+ // os.getenv('MYVAR', '') returns '' if MYVAR is not set but it's a bit more verbose,
203
+ // so setting errorOk to true will use the shorter version
204
+ errorOk = false,
205
+ ): string {
206
+ const reprFn = binary ? reprStrBinary : reprStr;
207
+ const reprs = [];
208
+ for (const t of word.tokens) {
209
+ if (typeof t === "string") {
210
+ reprs.push(reprFn(t));
211
+ } else if (t.type === "variable") {
212
+ // TODO: getenvb() is not available on Windows
213
+ const fn = binary ? "os.getenvb" : "os.getenv";
214
+ let getEnvCall = fn + "(" + reprFn(t.value);
215
+ if (!errorOk || word.tokens.length > 1) {
216
+ getEnvCall += ", " + reprFn("");
217
+ }
218
+ getEnvCall += ")";
219
+ reprs.push(getEnvCall);
220
+ // TODO: if the result of a repr() is discarded, this adds an unused import
221
+ imports.add("os");
222
+ } else if (t.type === "command") {
223
+ // TODO: warn that shell=True is a bad idea
224
+ // or properly parse the subcommand and render it as an array
225
+ let subprocessCall =
226
+ "subprocess.run(" +
227
+ reprStr(t.value) +
228
+ ", shell=True, capture_output=True";
229
+ if (!binary) {
230
+ subprocessCall += ", text=True";
231
+ }
232
+ subprocessCall += ").stdout";
233
+
234
+ // TODO: generate a descriptive command name with ChatGPT
235
+ // TODO: if there's only one command, name the variable "command" instead of "command1"
236
+ let i = 1;
237
+ let pyVar = "command" + i;
238
+ // We need to check because we often try to represent the same
239
+ // token twice and discard one of the attempts.
240
+ // This is linear time but hopefully there's not that many subcommands.
241
+ while (pyVar in osVars && osVars[pyVar] !== subprocessCall) {
242
+ i++;
243
+ pyVar = "command" + i;
244
+ if (i > Number.MAX_SAFE_INTEGER) {
245
+ throw new CCError("lol");
246
+ }
247
+ }
248
+ osVars[pyVar] = subprocessCall;
249
+ reprs.push(pyVar);
250
+ // TODO: if the result of a repr() is discarded, this adds an unused import
251
+ imports.add("subprocess");
252
+ }
253
+ }
254
+ return reprs.join(" + ");
255
+ }
256
+
257
+ function reprb(word: Word, osVars: OSVars, imports: Set<string>): string {
258
+ return repr(word, osVars, imports, true);
259
+ }
260
+
261
+ export function asFloat(
262
+ word: Word,
263
+ osVars: OSVars,
264
+ imports: Set<string>,
265
+ ): string {
266
+ if (word.isString()) {
267
+ // TODO: check it's actually a valid float
268
+ return word.toString();
269
+ }
270
+ return "float(" + repr(word, osVars, imports, false, true) + ")";
271
+ }
272
+
273
+ export function asInt(
274
+ word: Word,
275
+ osVars: OSVars,
276
+ imports: Set<string>,
277
+ ): string {
278
+ if (word.isString()) {
279
+ // TODO: check it's actually a valid int
280
+ return word.toString();
281
+ }
282
+ return "int(" + repr(word, osVars, imports, false, true) + ")";
283
+ }
284
+
285
+ // Port of Python's json.dumps() with its default options, which is what Requests uses
286
+ // https://github.com/psf/requests/blob/b0e025ade7ed30ed53ab61f542779af7e024932e/requests/models.py#L473
287
+ // It's different from JSON.stringify(). Namely, it adds spaces after ',' and ':'
288
+ // and all non-ASCII characters in strings are escaped:
289
+ // > JSON.stringify('\xFF')
290
+ // '"ÿ"'
291
+ // >>> json.dumps('\xFF')
292
+ // '"\\u00ff"'
293
+ const pythonJsonEscape = /"|\\|[^\x20-\x7E]/g;
294
+ function jsonRepr(s: string): string {
295
+ return (
296
+ '"' +
297
+ s.replace(pythonJsonEscape, (c: string): string => {
298
+ // https://tc39.es/ecma262/#table-json-single-character-escapes
299
+ switch (c) {
300
+ case "\b":
301
+ return "\\b";
302
+ case "\t":
303
+ return "\\t";
304
+ case "\n":
305
+ return "\\n";
306
+ case "\f":
307
+ return "\\f";
308
+ case "\r":
309
+ return "\\r";
310
+ case "\\":
311
+ return "\\\\";
312
+ case '"':
313
+ return '\\"';
314
+ }
315
+
316
+ const hex = c.charCodeAt(0).toString(16);
317
+ return "\\u" + hex.padStart(4, "0");
318
+ }) +
319
+ '"'
320
+ );
321
+ }
322
+
323
+ function ensure_minimum_exponent_length(n: string): string {
324
+ // If there's only 1 exponent digit, add a leading 0 to it
325
+ // ensure_minimum_exponent_length('1e-7') => '1e-07'
326
+ const [mantissa, exponent] = n.split("e");
327
+ const exponentSign = exponent[0];
328
+ const exponentValue = exponent.slice(1);
329
+ if (exponentValue.length === 1) {
330
+ return mantissa + "e" + exponentSign + "0" + exponentValue;
331
+ }
332
+ return n;
333
+ }
334
+ function floatAsPython(value: number): string {
335
+ // JSON.stringify() and lossless-json's stringify() don't stringify floats like Python.
336
+ // Notably, JavaScript doesn't add a trailing '.0' to floats that are integers but Python does
337
+ // JSON.stringify(10.0) => '10'
338
+ // str(10.0) => '10.0'
339
+ //
340
+ // Python adds a leading 0 to exponent notation numbers with 1 exponent digit
341
+ // JSON.stringify(1e-7) => '1e-7'
342
+ // str(1e-7) => '1e-07'
343
+ //
344
+ // Finally, Python will switch to scientific notation if the number has more than
345
+ // 17 digits not in scientific notation.
346
+ //
347
+ // Python's float formatting starts here:
348
+ // https://github.com/python/cpython/blob/bdc93b8a3563b4a3adb25fa902c0c879ccf427f6/Python/pystrtod.c#L915-L918
349
+ // and is ultimately this code:
350
+ // snprintf(buf, buf_size, "%.17g", val)
351
+ // change_decimal_from_locale_to_dot(buffer); // not important
352
+ // ensure_minimum_exponent_length(buffer, buf_size);
353
+ // ensure_decimal_point(buffer, buf_size, 17); // can switch to exponent notation
354
+ //
355
+ // And JavaScript's formatting is specified here:
356
+ // https://tc39.es/ecma262/multipage/ecmascript-data-types-and-values.html#sec-numeric-types-number-tostring
357
+ let asJsStr = value.toString();
358
+ if (asJsStr.includes("e")) {
359
+ asJsStr = ensure_minimum_exponent_length(asJsStr);
360
+ } else {
361
+ if (isInteger(asJsStr)) {
362
+ asJsStr += ".0";
363
+ }
364
+ // If there's more than 17 digits of precision, switch to scientific notation
365
+ const significantDigits = asJsStr
366
+ .replace(/^-/, "")
367
+ .replace(".", "")
368
+ .replace(/^0+/, "");
369
+ const asExponential = ensure_minimum_exponent_length(value.toExponential());
370
+ if (
371
+ significantDigits.length > 17 ||
372
+ (asExponential.length < asJsStr.length &&
373
+ asJsStr.split(".")[1].length > 4)
374
+ ) {
375
+ asJsStr = asExponential;
376
+ }
377
+ }
378
+ return asJsStr;
379
+ }
380
+
381
+ function jsonDumps(obj: string | number | boolean | object | null): string {
382
+ if (isLosslessNumber(obj)) {
383
+ const numAsStr = jsonStringifyLossless(obj) as string;
384
+ if (isInteger(numAsStr)) {
385
+ return numAsStr;
386
+ }
387
+
388
+ if (!isSafeNumber(numAsStr)) {
389
+ throw new CCError("float unrepresentable in Python: " + numAsStr);
390
+ }
391
+ // Can't be bigint because it's !isInteger and isSafeNumber
392
+ return floatAsPython(obj.valueOf() as number);
393
+ }
394
+
395
+ switch (typeof obj) {
396
+ case "string":
397
+ return jsonRepr(obj);
398
+ case "number":
399
+ // If the number in the JSON file is very large, it'll turn into Infinity
400
+ if (!isFinite(obj)) {
401
+ throw new CCError("found Infitiny in JSON");
402
+ }
403
+ // TODO: If the number in the JSON file is too big for JavaScript, we will lose information
404
+ // TODO: JavaScript and Python serialize floats differently.
405
+ // JSON.stringify(2e2) => 200
406
+ // json.dumps(2e2) => 200.0
407
+ return obj.toString();
408
+ case "boolean":
409
+ return obj.toString();
410
+ case "object":
411
+ if (obj === null) {
412
+ return "null";
413
+ }
414
+ if (Array.isArray(obj)) {
415
+ return "[" + obj.map(jsonDumps).join(", ") + "]";
416
+ }
417
+ return (
418
+ "{" +
419
+ Object.entries(obj)
420
+ .map((e) => jsonRepr(e[0]) + ": " + jsonDumps(e[1]))
421
+ .join(", ") +
422
+ "}"
423
+ );
424
+ default:
425
+ throw new CCError(
426
+ "unexpected object type that shouldn't appear in JSON: " + typeof obj,
427
+ );
428
+ }
429
+ }
430
+
431
+ function objToPython(
432
+ obj: string | number | boolean | object | null,
433
+ indent = 0,
434
+ ): string {
435
+ if (isLosslessNumber(obj)) {
436
+ const numAsStr = jsonStringifyLossless(obj) as string;
437
+ // If the number is a large float, it might not be representable in Python
438
+ // Both JavaScript and Python use f64 so we check if the float
439
+ // is representable in JavaScript.
440
+ if (!isInteger(numAsStr) && !isSafeNumber(numAsStr)) {
441
+ throw new CCError("float unrepresentable in Python: " + numAsStr);
442
+ }
443
+ // Displaying floats as they will be serialized in Python would help users
444
+ // understand why they're getting the "JSON won't be serialized as it was originally"
445
+ // message, but I think displaying them as they appear in the JSON is likely
446
+ // to be more convenient if you need to edit the value.
447
+ return numAsStr;
448
+ }
449
+
450
+ switch (typeof obj) {
451
+ case "string":
452
+ return reprStr(obj);
453
+ case "number":
454
+ // TODO: there are differences in number serialization between Python and JavaScript
455
+ // TODO: if the number in the JSON file is too big for JavaScript, we will lose information
456
+ return obj.toString();
457
+ case "boolean":
458
+ return obj ? "True" : "False";
459
+ case "object":
460
+ if (obj === null) {
461
+ return "None";
462
+ }
463
+ if (Array.isArray(obj)) {
464
+ if (obj.length === 0) {
465
+ return "[]";
466
+ }
467
+ let s = "[\n";
468
+ for (const item of obj) {
469
+ s += " ".repeat(indent + 4) + objToPython(item, indent + 4) + ",\n";
470
+ }
471
+ s += " ".repeat(indent) + "]";
472
+ return s;
473
+ }
474
+
475
+ if (Object.keys(obj).length === 0) {
476
+ return "{}";
477
+ }
478
+ {
479
+ let s = "{\n";
480
+ for (const [k, v] of Object.entries(obj)) {
481
+ // repr() because JSON keys must be strings.
482
+ s +=
483
+ " ".repeat(indent + 4) +
484
+ reprStr(k) +
485
+ ": " +
486
+ objToPython(v, indent + 4) +
487
+ ",\n";
488
+ }
489
+ s += " ".repeat(indent) + "}";
490
+ return s;
491
+ }
492
+ default:
493
+ throw new CCError(
494
+ "unexpected object type that shouldn't appear in JSON: " + typeof obj,
495
+ );
496
+ }
497
+ }
498
+
499
+ export function formatHeaders(
500
+ headers: Headers,
501
+ commentedOutHeaders: { [key: string]: string },
502
+ osVars: OSVars,
503
+ imports: Set<string>,
504
+ ): string {
505
+ // TODO: what if there are repeat headers
506
+ let headerDict = "headers = {\n";
507
+ for (const [headerName, headerValue] of headers) {
508
+ if (headerValue === null) {
509
+ continue;
510
+ }
511
+
512
+ let lineStart;
513
+ const headerNameLower = headerName.toLowerCase().toString();
514
+ if (has(commentedOutHeaders, headerNameLower)) {
515
+ if (commentedOutHeaders[headerNameLower]) {
516
+ headerDict += " # " + commentedOutHeaders[headerNameLower] + "\n";
517
+ }
518
+ lineStart = " # ";
519
+ } else {
520
+ lineStart = " ";
521
+ }
522
+ headerDict +=
523
+ lineStart +
524
+ repr(headerName, osVars, imports) +
525
+ ": " +
526
+ repr(headerValue, osVars, imports) +
527
+ ",\n";
528
+ }
529
+ headerDict += "}\n";
530
+ return headerDict;
531
+ }
532
+
533
+ function decodePercentEncoding(s: Word): Word | null {
534
+ let decoded;
535
+ try {
536
+ // https://url.spec.whatwg.org/#urlencoded-parsing recommends replacing + with space
537
+ // before decoding.
538
+ decoded = wordDecodeURIComponent(s.replace(/\+/g, " "));
539
+ } catch (e) {
540
+ if (e instanceof URIError) {
541
+ // String contains invalid percent encoded characters
542
+ return null;
543
+ }
544
+ throw e;
545
+ }
546
+ // If the query string doesn't round-trip, we cannot properly convert it.
547
+ const roundTripKey = percentEncode(decoded);
548
+ // If the original data used %20 instead of + (what requests will send), that's close enough
549
+ if (!eq(roundTripKey, s) && !eq(roundTripKey.replace(/%20/g, "+"), s)) {
550
+ return null;
551
+ }
552
+ return decoded;
553
+ }
554
+
555
+ function dataEntriesToDict(
556
+ dataEntries: Array<[string, string]>,
557
+ ): { [key: string]: Array<string> } | null {
558
+ // Group keys
559
+ // TODO: because keys can be code that reads from a file, those should not be considered the
560
+ // same key, for example what if that file is /dev/urandom.
561
+ // TODO: would we need to distinguish if /dev/urandom came from @/dev/urandom or from @-?
562
+ const asDict: { [key: string]: Array<string> } = {};
563
+ let prevKey = null;
564
+ for (const [key, val] of dataEntries) {
565
+ if (prevKey === key) {
566
+ asDict[key].push(val);
567
+ } else {
568
+ if (!Object.prototype.hasOwnProperty.call(asDict, key)) {
569
+ asDict[key] = [val];
570
+ } else {
571
+ // A repeated key with a different key between one of its repetitions
572
+ // means we can't represent these entries as a dictionary.
573
+ return null;
574
+ }
575
+ }
576
+ prevKey = key;
577
+ }
578
+
579
+ return asDict;
580
+ }
581
+
582
+ function dataEntriesToPython(dataEntries: Array<[string, string]>): string {
583
+ if (dataEntries.length === 0) {
584
+ return "''"; // This shouldn't happen
585
+ }
586
+
587
+ const entriesDict = dataEntriesToDict(dataEntries);
588
+ if (entriesDict !== null) {
589
+ if (Object.keys(entriesDict).length === 0) {
590
+ return "''"; // This shouldn't happen
591
+ }
592
+ let s = "{\n";
593
+ for (const [key, vals] of Object.entries(entriesDict)) {
594
+ s += " " + key + ": ";
595
+ if (vals.length === 0) {
596
+ s += "''"; // This shouldn't happen
597
+ } else if (vals.length === 1) {
598
+ s += vals[0];
599
+ } else {
600
+ s += "[\n";
601
+ for (const val of vals) {
602
+ s += " " + val + ",\n";
603
+ }
604
+ s += " ]";
605
+ }
606
+ s += ",\n";
607
+ }
608
+ s += "}";
609
+ return s;
610
+ }
611
+
612
+ let s = "[\n";
613
+ for (const entry of dataEntries) {
614
+ const [key, val] = entry;
615
+ s += " (" + key + ", " + val + "),\n";
616
+ }
617
+ s += "]";
618
+ return s;
619
+ }
620
+
621
+ function formatDataAsEntries(
622
+ dataArray: DataParam[],
623
+ osVars: OSVars,
624
+ imports: Set<string>,
625
+ variableName: "data" | "params" = "data",
626
+ ): [string, string] | null {
627
+ // This code is more complicated than you might expect because it needs
628
+ // to handle a --data-urlencode that reads from a file followed by --json
629
+ // because --json doesn't add an '&' before its value. Specifically, we
630
+ // have these cases:
631
+ //
632
+ // --data-urlencode @filename --json =value
633
+ // {open('filename').read(): 'value'}
634
+ //
635
+ // --data-urlencode @filename --json key=value
636
+ // {open('filename').read() + 'key': 'value'}
637
+ //
638
+ // --data-urlencode @filename --json key
639
+ // error
640
+ //
641
+ // --data-urlencode name@filename --json value
642
+ // {'name': open('filename').read() + 'value'}
643
+ //
644
+ // --data-urlencode name@filename --json key=value
645
+ // error
646
+ //
647
+ // --data-urlencode name@filename --json =blah
648
+ // error
649
+ //
650
+ // --data-urlencode adds an '&' before its value, so we don't have to worry about
651
+ // --json <foo> --data-urlencode <bar>
652
+ for (const d of dataArray) {
653
+ if (!(d instanceof Word) && d.filetype !== "urlencode") {
654
+ return null;
655
+ }
656
+ }
657
+
658
+ const dataEntries: Array<[string, string | null]> = [];
659
+ let percentWarn = new Word();
660
+ for (const [i, d] of dataArray.entries()) {
661
+ if (d instanceof Word) {
662
+ let newEntries = d.split("&");
663
+
664
+ const prevEntry = i > 0 ? dataEntries[dataEntries.length - 1] : null;
665
+ if (prevEntry !== null) {
666
+ // If there's a prevEntry, we can assume it came from --data-urlencode
667
+ // because it would be part of the current `d` string if it didn't.
668
+ const [first, ...rest] = newEntries;
669
+ if (first.includes("=") && prevEntry[1] === null) {
670
+ const [key, val] = first.split("=", 2);
671
+ const decodedKey = decodePercentEncoding(key);
672
+ if (decodedKey === null) {
673
+ return null;
674
+ }
675
+ const decodedVal = decodePercentEncoding(val);
676
+ if (decodedVal === null) {
677
+ return null;
678
+ }
679
+ if (key.toBool()) {
680
+ prevEntry[0] += " + " + repr(decodedKey, osVars, imports);
681
+ }
682
+ prevEntry[1] = repr(decodedVal, osVars, imports);
683
+
684
+ if (!percentWarn.toBool()) {
685
+ if (key.includes("%20")) {
686
+ percentWarn = key;
687
+ }
688
+ if (val.includes("%20")) {
689
+ percentWarn = val;
690
+ }
691
+ }
692
+ } else if (!first.includes("=") && prevEntry[1] !== null) {
693
+ if (first.toBool()) {
694
+ const decodedVal = decodePercentEncoding(first);
695
+ if (decodedVal === null) {
696
+ return null;
697
+ }
698
+ prevEntry[1] += " + " + repr(decodedVal, osVars, imports);
699
+
700
+ if (!percentWarn.toBool() && first.includes("%20")) {
701
+ percentWarn = first;
702
+ }
703
+ }
704
+ } else {
705
+ return null;
706
+ }
707
+ newEntries = rest;
708
+ }
709
+
710
+ for (const [j, entry] of newEntries.entries()) {
711
+ if (
712
+ entry.isEmpty() &&
713
+ j === newEntries.length - 1 &&
714
+ i !== dataArray.length - 1
715
+ ) {
716
+ // A --data-urlencoded should come next
717
+ continue;
718
+ }
719
+ if (!entry.includes("=")) {
720
+ return null;
721
+ }
722
+ const [key, val] = entry.split("=", 2);
723
+ const decodedKey = decodePercentEncoding(key);
724
+ if (decodedKey === null) {
725
+ return null;
726
+ }
727
+ const decodedVal = decodePercentEncoding(val);
728
+ if (decodedVal === null) {
729
+ return null;
730
+ }
731
+ dataEntries.push([
732
+ repr(decodedKey, osVars, imports),
733
+ repr(decodedVal, osVars, imports),
734
+ ]);
735
+
736
+ if (!percentWarn.toBool()) {
737
+ if (key.includes("%20")) {
738
+ percentWarn = key;
739
+ }
740
+ if (val.includes("%20")) {
741
+ percentWarn = val;
742
+ }
743
+ }
744
+ }
745
+
746
+ continue;
747
+ }
748
+
749
+ // TODO: I bet Python doesn't treat file paths identically to curl
750
+ const readFile = eq(d.filename, "-")
751
+ ? "sys.stdin.read()"
752
+ : "open(" + repr(d.filename, osVars, imports, false, true) + ").read()";
753
+
754
+ if (!d.name) {
755
+ dataEntries.push([readFile, null]);
756
+ } else {
757
+ // Curl doesn't percent encode the name but Requests will
758
+ if (!eq(d.name, percentEncode(d.name))) {
759
+ return null;
760
+ }
761
+ dataEntries.push([repr(d.name, osVars, imports), readFile]);
762
+ }
763
+ }
764
+
765
+ if (dataEntries.some((e) => e[1] === null)) {
766
+ return null;
767
+ }
768
+
769
+ return [
770
+ variableName +
771
+ " = " +
772
+ dataEntriesToPython(dataEntries as [string, string][]) +
773
+ "\n",
774
+ percentWarn.toString(),
775
+ ];
776
+ }
777
+
778
+ function formatDataAsStr(
779
+ dataArray: DataParam[],
780
+ imports: Set<string>,
781
+ osVars: OSVars,
782
+ variableName: "data" | "params" = "data",
783
+ ): [string, boolean] {
784
+ // If one of the arguments has to be binary, then they all have to be binary
785
+ // because we can't mix bytes and str.
786
+ // An argument has to be binary when the input command has
787
+ // --data-binary @filename
788
+ // otherwise we could generate code that will try to read an image file as text and error.
789
+ const binary = dataArray.some(
790
+ (d) => !(d instanceof Word) && d.filetype === "binary",
791
+ );
792
+ const reprFunc = binary ? reprb : repr;
793
+ const prefix = binary ? "b" : "";
794
+ const mode = binary ? ", 'rb'" : "";
795
+
796
+ // If we see a string with non-ASCII characters, or read from a file (which may contain
797
+ // non-ASCII characters), we convert the entire string to bytes at the end.
798
+ // curl sends bytes as-is, which is presumably in UTF-8, whereas Requests sends
799
+ // 0x80-0xFF as Latin-1 (as-is) and throws an error if it sees codepoints
800
+ // above 0xFF.
801
+ // TODO: is this actually helpful?
802
+ let encode = false;
803
+ let encodeOnSeparateLine = false;
804
+
805
+ const lines = [];
806
+
807
+ let extra = new Word();
808
+ let i, d;
809
+ for ([i, d] of dataArray.entries()) {
810
+ const op = i === 0 ? "=" : "+=";
811
+ let line = variableName + " " + op + " ";
812
+
813
+ if (i < dataArray.length - 1 && d instanceof Word && d.endsWith("&")) {
814
+ // Put the trailing '&' on the next line so that we don't have single '&'s on their own lines
815
+ extra = new Word("&");
816
+ d = d.slice(0, -1);
817
+ }
818
+
819
+ if (d instanceof Word) {
820
+ if (d.length) {
821
+ line += reprFunc(d, osVars, imports);
822
+ lines.push(line);
823
+ encode ||= d.test(/[^\x00-\x7F]/);
824
+ }
825
+ continue;
826
+ }
827
+
828
+ const { filetype, name, filename } = d;
829
+ if (filetype === "urlencode" && name) {
830
+ line += reprFunc(extra.add(name).append("="), osVars, imports) + " + ";
831
+ encodeOnSeparateLine = true; // we would need to add parentheses because of the +
832
+ } else if (extra.toBool()) {
833
+ line += reprFunc(extra, osVars, imports) + " + ";
834
+ encodeOnSeparateLine = true;
835
+ }
836
+ if (extra.toBool()) {
837
+ encodeOnSeparateLine = true; // we would need to add parentheses because of the +
838
+ }
839
+
840
+ let readFile = "";
841
+ if (eq(filename, "-")) {
842
+ readFile += binary ? "sys.stdin.buffer" : "sys.stdin";
843
+ imports.add("sys");
844
+ } else {
845
+ // TODO: if filename is a command, this won't work because unlike bash,
846
+ // Python won't remove the trailing newline from the result of a command
847
+ // we need to add .trim()
848
+ line =
849
+ "with open(" +
850
+ repr(filename, osVars, imports, false, true) +
851
+ mode +
852
+ ") as f:\n " +
853
+ line;
854
+ readFile += "f";
855
+ }
856
+ readFile += ".read()";
857
+ if (!["binary", "json", "urlencode"].includes(filetype)) {
858
+ readFile += `.replace(${prefix}'\\n', ${prefix}'').replace(${prefix}'\\r', ${prefix}'')`;
859
+ }
860
+
861
+ if (filetype === "urlencode") {
862
+ readFile = "quote_plus(" + readFile + ")";
863
+ if (binary) {
864
+ // quote_plus() always returns a string
865
+ readFile += ".encode()";
866
+ }
867
+ imports.add("urllib.parse.quote_plus");
868
+ } else {
869
+ // --data-urlencode files don't need to be encoded because
870
+ // they'll be percent-encoded and therefore ASCII-only
871
+ encode = true;
872
+ }
873
+
874
+ line += readFile;
875
+ lines.push(line);
876
+ extra = new Word();
877
+ }
878
+
879
+ if (binary) {
880
+ encode = false;
881
+ } else if (encode && lines.length === 1 && !encodeOnSeparateLine) {
882
+ lines[lines.length - 1] += ".encode()";
883
+ encode = false;
884
+ }
885
+
886
+ return [lines.join("\n") + "\n", encode];
887
+ }
888
+
889
+ export function formatDataAsJson(
890
+ d: DataParam,
891
+ imports: Set<string>,
892
+ osVars: OSVars,
893
+ ): [string | null, boolean] {
894
+ if (d instanceof Word) {
895
+ if (!d.isString()) {
896
+ // TODO: a JSON parser that allows shell variables
897
+ return [null, false];
898
+ }
899
+
900
+ // Try to parse using lossless-json first, then fall back to JSON.parse
901
+ // TODO: repeated dictionary keys are discarded
902
+ // https://github.com/josdejong/lossless-json/issues/244
903
+ let dataAsJson;
904
+ try {
905
+ // TODO: types
906
+ // https://github.com/josdejong/lossless-json/issues/245
907
+ dataAsJson = jsonParseLossless(d.toString()) as any;
908
+ } catch {
909
+ try {
910
+ dataAsJson = JSON.parse(d.toString());
911
+ } catch {
912
+ return [null, false];
913
+ }
914
+ }
915
+
916
+ try {
917
+ const jsonDataString = "json_data = " + objToPython(dataAsJson) + "\n";
918
+ // JSON might not be serialized by Python exactly as it was originally
919
+ // due to different whitespace, float formatting like extra + in exponent
920
+ // (1e100 vs 1e+100), different escape sequences in strings
921
+ // ("\/" vs "/" or "\u0008" vs "\b") or duplicate object keys.
922
+ let jsonRoundtrips = false;
923
+ try {
924
+ jsonRoundtrips = jsonDumps(dataAsJson) === d.tokens[0];
925
+ } catch {}
926
+ return [jsonDataString, jsonRoundtrips];
927
+ } catch {}
928
+ } else if (d.filetype === "json") {
929
+ let jsonDataString = "";
930
+ jsonDataString +=
931
+ "with open(" +
932
+ repr(d.filename, osVars, imports, false, true) +
933
+ ") as f:\n";
934
+ jsonDataString += " json_data = json.load(f)\n";
935
+ imports.add("json");
936
+ return [jsonDataString, false];
937
+ }
938
+
939
+ return [null, false];
940
+ }
941
+
942
+ function getDataString(
943
+ request: Request,
944
+ osVars: OSVars,
945
+ warnings: Warnings,
946
+ ): [string | null, boolean | null, string | null, Set<string>] {
947
+ const imports = new Set<string>();
948
+ if (!request.data || !request.dataArray) {
949
+ return [null, false, null, imports];
950
+ }
951
+
952
+ // There's 4 ways to pass data to Requests (in descending order of preference):
953
+ // a or dictionary/list as the json= argument
954
+ // a dictionary, or a list of tuples (if the dictionary would have duplicate keys) as the data= argument
955
+ // a string as data=
956
+ // bytes as data=
957
+
958
+ // We can pass json= if the data is valid JSON and we've specified json in the
959
+ // Content-Type header because passing json= will set that header.
960
+ //
961
+ // However, if there will be a mismatch between how the JSON is formatted
962
+ // we need to output a commented out version of the request with data= as well.
963
+ // This can happen when there's extra whitespace in the original data or
964
+ // because the JSON contains numbers that are too big to be stored in
965
+ // JavaScript or because there's objects with duplicate keys.
966
+ const contentType = request.headers.get("content-type");
967
+ let dataAsJson: string | null = null;
968
+ let jsonRoundtrips = false;
969
+ if (
970
+ request.dataArray.length === 1 &&
971
+ contentType &&
972
+ contentType.split(";")[0].toString().trim() === "application/json"
973
+ ) {
974
+ [dataAsJson, jsonRoundtrips] = formatDataAsJson(
975
+ request.dataArray[0],
976
+ imports,
977
+ osVars,
978
+ );
979
+ }
980
+ if (jsonRoundtrips) {
981
+ return [null, false, dataAsJson, imports];
982
+ }
983
+
984
+ // data= can't be a dict or a list of tuples (i.e. entries) when
985
+ // there is a @file from --data, --data-binary or --json (because they can contain an '&' which would get escaped)
986
+ // there is a --data-urlencode without a name= or name@
987
+ // if you split the input on & and there's a value that doesn't contain an = (e.g. --data "foo=bar&" or simply --data "&")
988
+ // there is a name or value that doesn't roundtrip through percent encoding
989
+ const dataAsEntries = formatDataAsEntries(request.dataArray, osVars, imports);
990
+ if (dataAsEntries !== null) {
991
+ const [dataEntries, percentWarn] = dataAsEntries;
992
+ if (
993
+ eq(
994
+ request.headers.get("content-type"),
995
+ "application/x-www-form-urlencoded",
996
+ ) &&
997
+ request.headers.length === 1
998
+ ) {
999
+ // Requests adds the header when you include a body
1000
+ // so if it's the only header, don't print the headers
1001
+ request.headers.delete("content-type");
1002
+ }
1003
+ if (percentWarn) {
1004
+ warnings.push([
1005
+ "percent-encoded-spaces-in-data",
1006
+ 'data contains spaces encoded by curl as "%20" which will be sent as "+" instead: ' +
1007
+ JSON.stringify(percentWarn),
1008
+ ]);
1009
+ }
1010
+ return [dataEntries, false, dataAsJson, imports];
1011
+ }
1012
+
1013
+ const [dataAsString, shouldEncode] = formatDataAsStr(
1014
+ request.dataArray,
1015
+ imports,
1016
+ osVars,
1017
+ );
1018
+ return [dataAsString, shouldEncode, dataAsJson, imports];
1019
+ }
1020
+
1021
+ function getFilesString(
1022
+ request: Request,
1023
+ osVars: OSVars,
1024
+ imports: Set<string>,
1025
+ ): [string, boolean] {
1026
+ let usesStdin = false;
1027
+ if (!request.multipartUploads) {
1028
+ return ["", usesStdin];
1029
+ }
1030
+
1031
+ const multipartUploads = request.multipartUploads.map((m) => {
1032
+ // https://github.com/psf/requests/blob/2d5517682b3b38547634d153cea43d48fbc8cdb5/requests/models.py#L117
1033
+ //
1034
+ // Requests's multipart syntax looks like this:
1035
+ // name/filename: content
1036
+ // name: open(filename/contentFile)
1037
+ // name: (filename, open(contentFile)[, contentType[, headers]]))
1038
+ const name = m.name ? repr(m.name, osVars, imports) : "None";
1039
+
1040
+ if (!("contentType" in m) && !("headers" in m) && !("encoder" in m)) {
1041
+ if (
1042
+ "contentFile" in m &&
1043
+ eq(m.contentFile, m.filename) &&
1044
+ !eq(m.contentFile, "-")
1045
+ ) {
1046
+ return [
1047
+ name,
1048
+ "open(" +
1049
+ repr(m.contentFile, osVars, imports, false, true) +
1050
+ ", 'rb')",
1051
+ ];
1052
+ }
1053
+ if ("content" in m && "filename" in m && eq(m.name, m.filename)) {
1054
+ return [name, repr(m.content, osVars, imports)];
1055
+ }
1056
+ }
1057
+
1058
+ const sentFilename =
1059
+ "filename" in m && m.filename
1060
+ ? repr(m.filename, osVars, imports)
1061
+ : "None";
1062
+ const tuple = [sentFilename];
1063
+ if ("contentFile" in m) {
1064
+ if (eq(m.contentFile, "-")) {
1065
+ // TODO: use piped stdin if we have it
1066
+ usesStdin = true;
1067
+ tuple.push("sys.stdin.buffer.read())");
1068
+ } else {
1069
+ tuple.push(
1070
+ "open(" +
1071
+ repr(m.contentFile, osVars, imports, false, true) +
1072
+ ", 'rb')",
1073
+ );
1074
+ }
1075
+ } else {
1076
+ tuple.push(repr(m.content, osVars, imports));
1077
+ }
1078
+ let addedContentType = false;
1079
+ if ("contentType" in m && m.contentType) {
1080
+ tuple.push(repr(m.contentType, osVars, imports));
1081
+ addedContentType = true;
1082
+ }
1083
+ if (
1084
+ ("headers" in m && m.headers) ||
1085
+ ("headerFiles" in m && m.headerFiles)
1086
+ ) {
1087
+ if (!addedContentType) {
1088
+ tuple.push("None");
1089
+ }
1090
+
1091
+ const headerArg = [];
1092
+ if (m.headers) {
1093
+ let headerDict = "{";
1094
+ for (const header of m.headers) {
1095
+ // TODO: can have multiple headers in one headers= argument?
1096
+ const [name, value] = header.split(":", 2);
1097
+ headerDict +=
1098
+ repr(name, osVars, imports) +
1099
+ ": " +
1100
+ repr(value.trimStart() || new Word(), osVars, imports) +
1101
+ ", ";
1102
+ }
1103
+ if (headerDict.endsWith(", ")) {
1104
+ headerDict = headerDict.slice(0, -2);
1105
+ }
1106
+ headerDict += "}";
1107
+ headerArg.push(headerDict);
1108
+ }
1109
+ if (m.headerFiles && m.headerFiles.length) {
1110
+ for (const headerFile of m.headerFiles) {
1111
+ headerArg.push(
1112
+ "read_headerfile(" +
1113
+ repr(headerFile, osVars, imports, false, true) +
1114
+ ")",
1115
+ );
1116
+ }
1117
+ imports.add("read_headerfile");
1118
+ }
1119
+ tuple.push(headerArg.join(" | "));
1120
+ }
1121
+ return [name, "(" + tuple.join(", ") + ")"];
1122
+ });
1123
+
1124
+ const multipartUploadsAsDict = Object.fromEntries(multipartUploads);
1125
+
1126
+ let filesString = "files = ";
1127
+ if (Object.keys(multipartUploadsAsDict).length === multipartUploads.length) {
1128
+ filesString += "{\n";
1129
+ for (const [multipartKey, multipartValue] of multipartUploads) {
1130
+ filesString += " " + multipartKey + ": " + multipartValue + ",\n";
1131
+ }
1132
+ filesString += "}\n";
1133
+ } else {
1134
+ filesString += "[\n";
1135
+ for (const [multipartKey, multipartValue] of multipartUploads) {
1136
+ filesString += " (" + multipartKey + ", " + multipartValue + "),\n";
1137
+ }
1138
+ filesString += "]\n";
1139
+ }
1140
+
1141
+ return [filesString, usesStdin];
1142
+ }
1143
+
1144
+ // Don't add indent/comment characters to empty lines, most importantly the last line
1145
+ // which will be empty when there's a trailing newline.
1146
+ function indent(s: string, level: number) {
1147
+ if (level === 0) {
1148
+ return s;
1149
+ }
1150
+ const begin = " ".repeat(level);
1151
+ return s
1152
+ .split("\n")
1153
+ .map((l) => (l.trim() ? begin + l : l))
1154
+ .join("\n");
1155
+ }
1156
+ function commentOut(s: string) {
1157
+ return s
1158
+ .split("\n")
1159
+ .map((l) => (l.trim() ? "#" + l : l))
1160
+ .join("\n");
1161
+ }
1162
+
1163
+ function uniqueWarn(
1164
+ seenWarnings: Set<string>,
1165
+ warnings: Warnings,
1166
+ warning: [string, string],
1167
+ ) {
1168
+ if (!seenWarnings.has(warning[0])) {
1169
+ seenWarnings.add(warning[0]);
1170
+ warnings.push(warning);
1171
+ }
1172
+ }
1173
+
1174
+ function joinArgs(args: string[]) {
1175
+ let s = "(";
1176
+ if (args.join("").length < 100) {
1177
+ s += args.join(", ");
1178
+ } else {
1179
+ s += "\n";
1180
+ for (const arg of args) {
1181
+ s += " " + arg + ",\n";
1182
+ }
1183
+ }
1184
+ return s + ")";
1185
+ }
1186
+
1187
+ function requestToPython(
1188
+ request: Request,
1189
+ warnings: Warnings = [],
1190
+ imports: Set<string>,
1191
+ thirdPartyImports: Set<string>,
1192
+ ): string {
1193
+ const osVars: OSVars = {};
1194
+ const commentedOutHeaders: { [key: string]: string } = {
1195
+ // TODO: add a warning why this should be commented out?
1196
+ "accept-encoding": "",
1197
+ "content-length": "",
1198
+ };
1199
+ // https://github.com/icing/blog/blob/main/curl_on_a_weekend.md
1200
+ if (eq(request.headers.get("te"), "trailers")) {
1201
+ commentedOutHeaders.te = "Requests doesn't support trailers";
1202
+ }
1203
+
1204
+ let cookieStr;
1205
+ let cookieFile: Word | null = null;
1206
+ if (request.cookies) {
1207
+ // TODO: handle duplicate cookie names
1208
+ cookieStr = "cookies = {\n";
1209
+ for (const [cookieName, cookieValue] of request.cookies) {
1210
+ cookieStr +=
1211
+ " " +
1212
+ repr(cookieName, osVars, imports) +
1213
+ ": " +
1214
+ repr(cookieValue, osVars, imports) +
1215
+ ",\n";
1216
+ }
1217
+ cookieStr += "}\n";
1218
+ // Before Python 3.11, cookies= was sorted alphabetically
1219
+ // https://github.com/python/cpython/issues/86232
1220
+ commentedOutHeaders.cookie = "";
1221
+ if (request.cookieFiles) {
1222
+ warnings.push([
1223
+ "cookie-files",
1224
+ "passing both cookies and cookie files with --cookie/-b is not supported",
1225
+ ]);
1226
+ }
1227
+ if (request.cookieJar) {
1228
+ warnings.push([
1229
+ "cookie-files",
1230
+ "passing both cookies and --cookie-jar/-c is not supported",
1231
+ ]);
1232
+ }
1233
+ } else if (
1234
+ (request.cookieFiles && request.cookieFiles.length) ||
1235
+ request.cookieJar
1236
+ ) {
1237
+ imports.add("http.cookiejar.MozillaCookieJar");
1238
+ if (request.cookieFiles && request.cookieFiles.length) {
1239
+ // TODO: what if user passes multiple cookie files?
1240
+ // TODO: what if user passes cookies and cookie files?
1241
+ cookieFile = request.cookieFiles[request.cookieFiles.length - 1];
1242
+ if (request.cookieFiles.length > 1) {
1243
+ warnings.push([
1244
+ "cookie-files",
1245
+ // TODO: curl reads all of them.
1246
+ "multiple cookie files are not supported, using the last one: " +
1247
+ JSON.stringify(cookieFile.toString()),
1248
+ ]);
1249
+ }
1250
+ // TODO: do we need to .load()?
1251
+ cookieStr =
1252
+ "cookies = MozillaCookieJar(" +
1253
+ repr(cookieFile, osVars, imports) +
1254
+ ")\n";
1255
+ } else if (request.cookieJar) {
1256
+ cookieStr = "cookies = MozillaCookieJar()\n";
1257
+ }
1258
+ }
1259
+
1260
+ let proxyDict;
1261
+ if (request.proxy) {
1262
+ let proxy = request.proxy.includes("://")
1263
+ ? request.proxy
1264
+ : request.proxy.prepend("http://");
1265
+ const protocol = proxy.split("://")[0].toLowerCase();
1266
+ if (eq(protocol, "socks")) {
1267
+ // https://github.com/curl/curl/blob/curl-7_86_0/lib/url.c#L2418-L2419
1268
+ proxy = proxy.replace("socks", "socks4");
1269
+ }
1270
+ proxyDict = "proxies = {\n";
1271
+ proxyDict += " 'http': " + repr(proxy, osVars, imports) + ",\n";
1272
+ // TODO: if (protocol !== "http") { ?
1273
+ proxyDict += " 'https': " + repr(proxy, osVars, imports) + ",\n";
1274
+ proxyDict += "}\n";
1275
+ }
1276
+
1277
+ let certStr;
1278
+ if (request.cert || request.key) {
1279
+ certStr = "cert = ";
1280
+ let certPart = "None";
1281
+ if (request.cert) {
1282
+ const [cert, password] = request.cert;
1283
+ certPart = repr(cert, osVars, imports);
1284
+ if (password) {
1285
+ warnings.push([
1286
+ "cert-password",
1287
+ "passing a password to --cert is not supported: " +
1288
+ JSON.stringify(password.toString()),
1289
+ ]);
1290
+ }
1291
+ }
1292
+ if (request.key) {
1293
+ certStr +=
1294
+ "(" + certPart + ", " + repr(request.key, osVars, imports) + ")";
1295
+ } else {
1296
+ certStr += certPart;
1297
+ }
1298
+ certStr += "\n";
1299
+ }
1300
+
1301
+ // if there's only 1 URL, put params all together here, unless it's just one string.
1302
+ // if there's more than 1, if we have params that are added to each URL from
1303
+ // --get --data or --url-query that need to read a file, put just the shared part
1304
+ // here, then keep the query in the URL, in the URL.
1305
+ // If there's no --get --data or --url-query, then
1306
+ // put params (if it can be rendered as a list or dict) right before the requests line
1307
+ // Otherwise, keep the query in the URL.
1308
+ let paramsStr;
1309
+ let shouldEncodeParams; // TODO: necessary?
1310
+ function readsFile(paramArray: DataParam[]) {
1311
+ return paramArray.some((p) => !(p instanceof Word));
1312
+ }
1313
+ const paramArray =
1314
+ request.urls.length === 1 ? request.urls[0].queryArray : request.queryArray;
1315
+ if (
1316
+ paramArray &&
1317
+ (request.urls.length === 1 ||
1318
+ (request.urls.length > 1 && readsFile(paramArray)))
1319
+ ) {
1320
+ const queryAsEntries = formatDataAsEntries(
1321
+ paramArray,
1322
+ osVars,
1323
+ imports,
1324
+ "params",
1325
+ );
1326
+ if (queryAsEntries !== null) {
1327
+ let percentWarn;
1328
+ [paramsStr, percentWarn] = queryAsEntries;
1329
+
1330
+ if (percentWarn) {
1331
+ warnings.push([
1332
+ "percent-encoded-spaces-in-query",
1333
+ // TODO: will they?
1334
+ 'URL querystring contains spaces encoded as "%20" which will be sent by Requests as "+" instead: ' +
1335
+ JSON.stringify(percentWarn),
1336
+ ]);
1337
+ }
1338
+ } else if (readsFile(paramArray)) {
1339
+ [paramsStr, shouldEncodeParams] = formatDataAsStr(
1340
+ paramArray,
1341
+ imports,
1342
+ osVars,
1343
+ "params",
1344
+ );
1345
+ }
1346
+ }
1347
+
1348
+ const contentType = request.headers.get("content-type");
1349
+ let dataString;
1350
+ let jsonDataString;
1351
+ let filesString;
1352
+ let shouldEncode;
1353
+ if (request.urls[0].uploadFile && request.urls.length === 1) {
1354
+ // TODO: https://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
1355
+ if (
1356
+ eq(request.urls[0].uploadFile, "-") ||
1357
+ eq(request.urls[0].uploadFile, ".")
1358
+ ) {
1359
+ dataString = "data = sys.stdin.buffer.read()\n";
1360
+ imports.add("sys");
1361
+ } else {
1362
+ dataString =
1363
+ "with open(" +
1364
+ repr(request.urls[0].uploadFile, osVars, imports, false, true) +
1365
+ ", 'rb') as f:\n";
1366
+ dataString += " data = f.read()\n";
1367
+ }
1368
+ } else if (request.multipartUploads) {
1369
+ let usesStdin = false;
1370
+ [filesString, usesStdin] = getFilesString(request, osVars, imports);
1371
+ if (usesStdin) {
1372
+ imports.add("sys");
1373
+ }
1374
+ // If you pass files= then Requests adds this header and a `boundary`
1375
+ // If you manually pass a Content-Type header it won't set a `boundary`
1376
+ // wheras curl does, so the request will fail.
1377
+ // https://github.com/curlconverter/curlconverter/issues/248
1378
+ if (
1379
+ filesString &&
1380
+ contentType &&
1381
+ eq(contentType.trim(), "multipart/form-data") &&
1382
+ !contentType.includes("boundary=")
1383
+ ) {
1384
+ // TODO: better wording
1385
+ commentedOutHeaders["content-type"] =
1386
+ "requests won't add a boundary if this header is set when you pass files=";
1387
+ }
1388
+ } else if (request.data && !request.data.isEmpty()) {
1389
+ // !isEmpty() because passing data='' is the same as not passing data=
1390
+ // We need to set the Content-Type header in headers= and not set data=
1391
+ let dataImports: Set<string>;
1392
+ [dataString, shouldEncode, jsonDataString, dataImports] = getDataString(
1393
+ request,
1394
+ osVars,
1395
+ warnings,
1396
+ );
1397
+ dataImports.forEach(imports.add, imports);
1398
+ // Remove "Content-Type" from the headers dict
1399
+ // because Requests adds it automatically when you use json=
1400
+ if (
1401
+ jsonDataString &&
1402
+ !dataString &&
1403
+ contentType &&
1404
+ eq(contentType.trim(), "application/json")
1405
+ ) {
1406
+ commentedOutHeaders["content-type"] = "Already added when you pass json=";
1407
+ }
1408
+ }
1409
+
1410
+ let headerDict;
1411
+ if (request.headers.length) {
1412
+ headerDict = formatHeaders(
1413
+ request.headers,
1414
+ commentedOutHeaders,
1415
+ osVars,
1416
+ imports,
1417
+ );
1418
+ }
1419
+
1420
+ let pythonCode = "";
1421
+
1422
+ if (proxyDict) {
1423
+ pythonCode += proxyDict + "\n";
1424
+ }
1425
+ if (cookieStr) {
1426
+ pythonCode += cookieStr + "\n";
1427
+ }
1428
+ if (headerDict) {
1429
+ pythonCode += headerDict + "\n";
1430
+ }
1431
+ if (paramsStr) {
1432
+ pythonCode += paramsStr + "\n";
1433
+ }
1434
+ if (certStr) {
1435
+ pythonCode += certStr + "\n";
1436
+ }
1437
+ if (jsonDataString) {
1438
+ pythonCode += jsonDataString + "\n";
1439
+ } else if (dataString) {
1440
+ pythonCode += dataString + "\n";
1441
+ } else if (filesString) {
1442
+ pythonCode += filesString + "\n";
1443
+ }
1444
+
1445
+ // By default, curl doesn't follow redirects and Requests does.
1446
+ // Unless redirect behavior has been explicitly set with -L/--location/--no-location
1447
+ // or --max-redirs 0 we pretend generate code that follows redirects,
1448
+ // because adding allow_redirects=False to almost every command would be ugly
1449
+ // and it only matters when the server responds with a redirect, which isn't
1450
+ // that common.
1451
+ let followRedirects = request.followRedirects;
1452
+ let maxRedirects = undefined;
1453
+ if (request.maxRedirects !== undefined) {
1454
+ maxRedirects = asInt(request.maxRedirects, osVars, imports);
1455
+ }
1456
+ if (followRedirects === undefined) {
1457
+ followRedirects = true;
1458
+
1459
+ // Users would see this warning for most commands
1460
+ // warnings.push([
1461
+ // "--location",
1462
+ // "Requests defaults to following redirects, curl doesn't",
1463
+ // ]);
1464
+ }
1465
+ const hasMaxRedirects =
1466
+ followRedirects &&
1467
+ maxRedirects &&
1468
+ maxRedirects !== "0" &&
1469
+ maxRedirects !== "30"; // Requests default
1470
+
1471
+ // Things that vary per-url:
1472
+ // method (because --upload-file can make it PUT)
1473
+ // data= (because of --upload-file)
1474
+ // --output file
1475
+ // params= (because of the query string)
1476
+ // auth= (because the URL can have an auth string)
1477
+ const seenWarnings: Set<string> = new Set();
1478
+ const requestLines = [];
1479
+ let extraEmptyLine = false;
1480
+ for (const [urlObjIndex, urlObj] of request.urls.entries()) {
1481
+ const requestsMethods = [
1482
+ "GET",
1483
+ "HEAD",
1484
+ "POST",
1485
+ "PATCH",
1486
+ "PUT",
1487
+ "DELETE",
1488
+ "OPTIONS", // undocumented
1489
+ ];
1490
+ let fn;
1491
+ const args = [];
1492
+ const methodAsString = urlObj.method.toString();
1493
+ if (urlObj.method.isString() && requestsMethods.includes(methodAsString)) {
1494
+ fn = methodAsString.toLowerCase();
1495
+ } else {
1496
+ fn = "request";
1497
+ args.push(repr(urlObj.method, osVars, imports));
1498
+
1499
+ if (
1500
+ urlObj.method.isString() &&
1501
+ methodAsString !== methodAsString.toUpperCase()
1502
+ ) {
1503
+ warnings.push([
1504
+ "method",
1505
+ "Requests will uppercase the HTTP method: " +
1506
+ JSON.stringify(methodAsString),
1507
+ ]);
1508
+ }
1509
+ }
1510
+
1511
+ let urlParamsStr;
1512
+ let url = urlObj.url;
1513
+ if (request.urls.length === 1) {
1514
+ if (paramsStr) {
1515
+ url = urlObj.urlWithoutQueryArray;
1516
+ } else {
1517
+ url = urlObj.url;
1518
+ }
1519
+ } else {
1520
+ if (paramsStr) {
1521
+ url = urlObj.urlWithOriginalQuery;
1522
+ } else {
1523
+ if (urlObj.queryArray && urlObj.queryArray.length > 0) {
1524
+ const urlQueryAsEntries = formatDataAsEntries(
1525
+ urlObj.queryArray,
1526
+ osVars,
1527
+ imports,
1528
+ "params",
1529
+ );
1530
+ if (urlQueryAsEntries !== null) {
1531
+ let percentWarn;
1532
+ [urlParamsStr, percentWarn] = urlQueryAsEntries;
1533
+ url = urlObj.urlWithoutQueryArray;
1534
+
1535
+ if (percentWarn) {
1536
+ warnings.push([
1537
+ "percent-encoded-spaces-in-query",
1538
+ // TODO: will they?
1539
+ 'query contains spaces encoded by curl as "%20" which will be sent as "+" instead: ' +
1540
+ JSON.stringify(percentWarn),
1541
+ ]);
1542
+ }
1543
+ } else if (readsFile(urlObj.queryArray)) {
1544
+ [urlParamsStr, shouldEncodeParams] = formatDataAsStr(
1545
+ urlObj.queryArray,
1546
+ imports,
1547
+ osVars,
1548
+ "params",
1549
+ );
1550
+ url = urlObj.urlWithoutQueryArray;
1551
+ }
1552
+ }
1553
+ // url = urlObj.url
1554
+ }
1555
+ }
1556
+ args.push(repr(url, osVars, imports));
1557
+
1558
+ if (paramsStr || urlParamsStr) {
1559
+ args.push("params=params" + (shouldEncodeParams ? ".encode()" : ""));
1560
+ }
1561
+ if (cookieStr && !request.cookieJar) {
1562
+ args.push("cookies=cookies");
1563
+ }
1564
+ if (headerDict) {
1565
+ args.push("headers=headers");
1566
+ }
1567
+ if (urlObj.uploadFile) {
1568
+ if (request.urls.length > 1) {
1569
+ // If there's more than one URL we could have --data for all
1570
+ // of them and --upload-file for just one of them and we can't
1571
+ // overwrite the `data` variable in that case.
1572
+ args.push("data=file_contents");
1573
+ } else {
1574
+ args.push("data=data");
1575
+ }
1576
+ } else if (filesString) {
1577
+ args.push("files=files");
1578
+ } else if (request.data && !request.data.isEmpty()) {
1579
+ if (jsonDataString) {
1580
+ args.push("json=json_data");
1581
+ } else {
1582
+ args.push("data=data" + (shouldEncode ? ".encode()" : ""));
1583
+ }
1584
+ }
1585
+ if (proxyDict) {
1586
+ args.push("proxies=proxies");
1587
+ }
1588
+ if (certStr) {
1589
+ args.push("cert=cert");
1590
+ }
1591
+ const certOrPath = request.cacert || request.capath;
1592
+ if (request.insecure) {
1593
+ args.push("verify=False");
1594
+ } else if (certOrPath) {
1595
+ args.push("verify=" + repr(certOrPath, osVars, imports));
1596
+ }
1597
+ // TODO: does this header check apply to all auth methods?
1598
+ if (urlObj.auth && !request.headers.has("Authorization")) {
1599
+ const [user, password] = urlObj.auth;
1600
+ let auth =
1601
+ "(" +
1602
+ repr(user, osVars, imports) +
1603
+ ", " +
1604
+ repr(password, osVars, imports) +
1605
+ ")";
1606
+ switch (request.authType) {
1607
+ case "basic":
1608
+ break;
1609
+ case "digest":
1610
+ thirdPartyImports.add("requests.auth.HTTPDigestAuth");
1611
+ auth = "HTTPDigestAuth" + auth;
1612
+ break;
1613
+ case "ntlm":
1614
+ case "ntlm-wb":
1615
+ thirdPartyImports.add("requests_ntlm.HttpNtlmAuth");
1616
+ auth = "HttpNtlmAuth" + auth;
1617
+ // TODO: this could stop being true
1618
+ uniqueWarn(seenWarnings, warnings, [
1619
+ "ntlm",
1620
+ "requests-ntlm is unmaintained",
1621
+ ]);
1622
+ break;
1623
+ case "negotiate":
1624
+ thirdPartyImports.add("requests_gssapi.HTTPSPNEGOAuth");
1625
+ auth = "HTTPSPNEGOAuth(";
1626
+ if (request.delegation) {
1627
+ if (eq(request.delegation, "always")) {
1628
+ auth += "delegate=True";
1629
+ } else if (eq(request.delegation, "none")) {
1630
+ auth += "delegate=False";
1631
+ } else {
1632
+ uniqueWarn(seenWarnings, warnings, [
1633
+ "delegation",
1634
+ "--delegation value not supported: " +
1635
+ JSON.stringify(request.delegation.toString()),
1636
+ ]);
1637
+ }
1638
+ }
1639
+ auth += ")";
1640
+ // TODO: use requests-kerberos instead?
1641
+ // https://star-history.com/#pythongssapi/requests-gssapi&requests/requests-kerberos
1642
+ uniqueWarn(seenWarnings, warnings, [
1643
+ "negotiate",
1644
+ "requests-gssapi is a fork of requests-kerberos",
1645
+ ]);
1646
+ break;
1647
+ case "aws-sigv4":
1648
+ thirdPartyImports.add("aws_requests_auth.aws_auth.AWSRequestsAuth");
1649
+ // TODO: move this "auth = " to separate line
1650
+ // TODO: try this.
1651
+ auth =
1652
+ "AWSRequestsAuth(aws_access_key=" +
1653
+ repr(user, osVars, imports) +
1654
+ ", aws_secret_access_key=" +
1655
+ repr(password, osVars, imports) +
1656
+ // TODO: parse
1657
+ ", aws_host=" +
1658
+ repr(request.awsSigV4 || new Word(), osVars, imports) +
1659
+ ", aws_region=" +
1660
+ repr(request.awsSigV4 || new Word(), osVars, imports) +
1661
+ ", aws_service=" +
1662
+ repr(request.awsSigV4 || new Word(), osVars, imports) +
1663
+ ")";
1664
+ uniqueWarn(seenWarnings, warnings, [
1665
+ "--aws-sigv4",
1666
+ "--aws-sigv4 value isn't parsed: " +
1667
+ JSON.stringify(
1668
+ request.awsSigV4 ? request.awsSigV4.toString() : "",
1669
+ ),
1670
+ ]);
1671
+ break;
1672
+ case "bearer":
1673
+ // Shouldn't happen because hasHeader(Authorization) should be true
1674
+ // TODO: use requests-oauthlib
1675
+ break;
1676
+ }
1677
+ args.push("auth=" + auth);
1678
+ }
1679
+ if (request.timeout || request.connectTimeout) {
1680
+ let connectTimeout = null;
1681
+ if (request.connectTimeout) {
1682
+ connectTimeout = asFloat(request.connectTimeout, osVars, imports);
1683
+ }
1684
+ let timeout = null;
1685
+ if (request.timeout) {
1686
+ if (request.timeout.isString()) {
1687
+ timeout = request.timeout.toString();
1688
+ } else {
1689
+ timeout =
1690
+ "float(" +
1691
+ repr(request.timeout, osVars, imports, false, true) +
1692
+ ")";
1693
+ }
1694
+ }
1695
+
1696
+ if (timeout && connectTimeout && timeout !== connectTimeout) {
1697
+ args.push("timeout=(" + connectTimeout + ", " + timeout + ")");
1698
+ } else if (timeout) {
1699
+ args.push("timeout=" + timeout);
1700
+ } else if (connectTimeout) {
1701
+ args.push("timeout=(" + connectTimeout + ", None)");
1702
+ }
1703
+
1704
+ if (timeout) {
1705
+ uniqueWarn(seenWarnings, warnings, [
1706
+ "--max-time",
1707
+ // https://requests.readthedocs.io/en/latest/user/advanced/#timeouts
1708
+ "unlike --max-time, Requests doesn't have a timeout for the whole request, only for the connect and the read",
1709
+ ]);
1710
+ }
1711
+ }
1712
+
1713
+ if (!followRedirects || maxRedirects === "0") {
1714
+ args.push("allow_redirects=False");
1715
+ } else if (maxRedirects) {
1716
+ if (maxRedirects === "-1") {
1717
+ imports.add("math");
1718
+ maxRedirects = "math.inf";
1719
+ }
1720
+ }
1721
+ if (followRedirects && request.followRedirectsTrusted) {
1722
+ uniqueWarn(seenWarnings, warnings, [
1723
+ "--location-trusted",
1724
+ "Requests doesn't have an easy way to disable removing the Authorization: header on redirect",
1725
+ ]);
1726
+ }
1727
+
1728
+ let requestLine = "";
1729
+ const isSession = hasMaxRedirects || request.cookieJar;
1730
+ const indentLevel = isSession ? 1 : 0;
1731
+
1732
+ if (isSession && urlObjIndex === 0) {
1733
+ requestLine += "with requests.Session() as session:\n";
1734
+ if (hasMaxRedirects) {
1735
+ requestLine += ` session.max_redirects = ${maxRedirects}\n`;
1736
+ }
1737
+ if (request.cookieJar) {
1738
+ requestLine += ` session.cookies = cookies\n`;
1739
+ }
1740
+ }
1741
+
1742
+ if (request.urls.length > 1) {
1743
+ if (urlParamsStr) {
1744
+ requestLine += indent(urlParamsStr, indentLevel);
1745
+ }
1746
+
1747
+ if (urlObj.uploadFile) {
1748
+ let uploadFileLine = "";
1749
+ // TODO: https://docs.python-requests.org/en/latest/user/advanced/#streaming-uploads
1750
+ if (eq(urlObj.uploadFile, "-") || eq(urlObj.uploadFile, ".")) {
1751
+ uploadFileLine += "file_contents = sys.stdin.buffer.read()\n";
1752
+ imports.add("sys");
1753
+ } else {
1754
+ uploadFileLine +=
1755
+ "with open(" +
1756
+ repr(urlObj.uploadFile, osVars, imports, false, true) +
1757
+ ", 'rb') as f:\n";
1758
+ uploadFileLine += " file_contents = f.read()\n";
1759
+ }
1760
+ requestLine += indent(uploadFileLine, indentLevel);
1761
+ }
1762
+ }
1763
+
1764
+ const fnToCall =
1765
+ "response = " + (isSession ? "session" : "requests") + "." + fn;
1766
+ requestLine += indent(fnToCall + joinArgs(args) + "\n", indentLevel);
1767
+
1768
+ if (jsonDataString && dataString && !urlObj.uploadFile) {
1769
+ // Adding empty lines to a "with" block breaks the code when pasted in the REPL
1770
+ requestLine += isSession || request.urls.length > 1 ? "" : "\n";
1771
+
1772
+ // Should never be -1
1773
+ args[args.indexOf("json=json_data")] = shouldEncode
1774
+ ? "data=data.encode()"
1775
+ : "data=data";
1776
+ let dataAlternative =
1777
+ "# Note: json_data will not be serialized by requests\n" +
1778
+ "# exactly as it was in the original request.\n";
1779
+ dataAlternative += commentOut(dataString);
1780
+ dataAlternative += commentOut(fnToCall + joinArgs(args) + "\n");
1781
+ requestLine += indent(dataAlternative, indentLevel);
1782
+ }
1783
+
1784
+ if (urlObj.output && !eq(urlObj.output, "/dev/null")) {
1785
+ let outputLine = "";
1786
+ if (eq(request.urls[0].output, "-")) {
1787
+ outputLine += "print(response.text)\n";
1788
+ } else {
1789
+ outputLine += isSession || request.urls.length > 1 ? "" : "\n";
1790
+
1791
+ outputLine +=
1792
+ "with open(" +
1793
+ repr(urlObj.output, osVars, imports, false, true) +
1794
+ ", 'wb') as f:\n";
1795
+ outputLine += " f.write(response.content)\n";
1796
+ }
1797
+ requestLine += indent(outputLine, indentLevel);
1798
+ }
1799
+
1800
+ if (
1801
+ !isSession &&
1802
+ // request.urls.length > 1 &&
1803
+ (urlParamsStr ||
1804
+ (dataString && jsonDataString) ||
1805
+ urlObj.uploadFile ||
1806
+ (urlObj.output && !eq(urlObj.output, "/dev/null")))
1807
+ ) {
1808
+ extraEmptyLine = true;
1809
+ }
1810
+ requestLines.push(requestLine);
1811
+ }
1812
+
1813
+ pythonCode += requestLines.join(extraEmptyLine ? "\n" : "");
1814
+
1815
+ if (request.cookieJar) {
1816
+ let cookieSaveLine = "cookies.save(";
1817
+ if (!eq(request.cookieJar, cookieFile)) {
1818
+ cookieSaveLine += repr(request.cookieJar, osVars, imports) + ", ";
1819
+ }
1820
+ cookieSaveLine += "ignore_discard=True, ignore_expires=True)\n"; // TODO: necessary?
1821
+ pythonCode += indent(cookieSaveLine, 1);
1822
+ }
1823
+
1824
+ if (request.http3) {
1825
+ warnings.push([
1826
+ "http3",
1827
+ "this was an HTTP/3 request but requests only supports HTTP/1.1",
1828
+ ]);
1829
+ } else if (request.http2) {
1830
+ warnings.push([
1831
+ "http2",
1832
+ "this was an HTTP/2 request but requests only supports HTTP/1.1",
1833
+ ]);
1834
+ } else if (request.headers.lowercase && request.headers.length > 1) {
1835
+ warnings.push([
1836
+ "--header",
1837
+ "all the --header/-H names are lowercase, which means this may have been an HTTP/2 or HTTP/3 request. Requests only sends HTTP/1.1",
1838
+ ]);
1839
+ }
1840
+
1841
+ // It would be more correct to run the commands whenever we need
1842
+ // their values instead of running them all at the beginning.
1843
+ let variableCode = "";
1844
+ if (Object.keys(osVars).length) {
1845
+ for (const [varName, expr] of Object.entries(osVars)) {
1846
+ variableCode += `${varName} = ${expr}\n`;
1847
+ }
1848
+ variableCode += "\n";
1849
+ }
1850
+
1851
+ return variableCode + pythonCode;
1852
+ }
1853
+
1854
+ export function printImports(imps: Set<string>): string {
1855
+ let s = "";
1856
+ for (const imp of Array.from(imps).sort()) {
1857
+ if (imp === "read_headerfile") {
1858
+ continue;
1859
+ }
1860
+ if (imp.includes(".")) {
1861
+ const pos = imp.lastIndexOf(".");
1862
+ const module = imp.slice(0, pos);
1863
+ const name = imp.slice(pos + 1);
1864
+ s += "from " + module + " import " + name + "\n";
1865
+ } else {
1866
+ s += "import " + imp + "\n";
1867
+ }
1868
+ }
1869
+ return s;
1870
+ }
1871
+
1872
+ export function _toPython(
1873
+ requests: Request[],
1874
+ warnings: Warnings = [],
1875
+ ): string {
1876
+ const code = [];
1877
+ let joinTwoLines = false;
1878
+ const imports = new Set<string>();
1879
+ const thirdPartyImports = new Set<string>();
1880
+ for (const request of requests) {
1881
+ const requestCode = requestToPython(
1882
+ request,
1883
+ warnings,
1884
+ imports,
1885
+ thirdPartyImports,
1886
+ );
1887
+ code.push(requestCode);
1888
+
1889
+ // If one of the requests defines variables (or its URL is very long),
1890
+ // separate all the configs with two empty lines.
1891
+ // curl example.com --next example.com
1892
+ // vs.
1893
+ // curl --data "foo=bar" example.com --next example.com
1894
+ // (+1 for the trailing newline)
1895
+ joinTwoLines ||= requestCode.split("\n").length > request.urls.length + 1;
1896
+ }
1897
+
1898
+ let importCode = "";
1899
+ importCode += printImports(imports);
1900
+ if (imports.size > 1) {
1901
+ importCode += "\n";
1902
+ }
1903
+ importCode += "import requests\n";
1904
+ importCode += printImports(thirdPartyImports);
1905
+ importCode += "\n";
1906
+ if (imports.has("read_headerfile")) {
1907
+ importCode += "def read_headerfile(filename):\n";
1908
+ importCode += " headers = {}\n";
1909
+ importCode += " with open(filename) as f:\n";
1910
+ importCode += " for line in f:\n";
1911
+ importCode += " line = line.split('#', 1)[0].strip()\n";
1912
+ importCode += " if line:\n";
1913
+ importCode += " [key, value] = line.split(':', 1)\n";
1914
+ importCode += " value = value or ''\n";
1915
+ importCode += " headers[key.strip()] = value.strip()\n";
1916
+ importCode += " return headers\n";
1917
+ importCode += "\n";
1918
+ }
1919
+
1920
+ return importCode + code.join(joinTwoLines ? "\n\n" : "\n");
1921
+ }
1922
+
1923
+ export function toPythonWarn(
1924
+ curlCommand: string | string[],
1925
+ warnings: Warnings = [],
1926
+ ): [string, Warnings] {
1927
+ const requests = parse(curlCommand, supportedArgs, warnings);
1928
+ const python = _toPython(requests, warnings);
1929
+ return [python, warnings];
1930
+ }
1931
+
1932
+ export function toPython(curlCommand: string | string[]): string {
1933
+ return toPythonWarn(curlCommand)[0];
1934
+ }