@sasjs/utils 2.49.0 → 2.49.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/fs/internal/helper.js +19 -24
- package/fs/internal/helper.js.map +1 -1
- package/macros/mf_abort.sas +27 -0
- package/macros/mf_dedup.sas +53 -0
- package/macros/mf_deletefile.sas +31 -0
- package/macros/mf_existds.sas +30 -0
- package/macros/mf_existfeature.sas +59 -0
- package/macros/mf_existfileref.sas +33 -0
- package/macros/mf_existfunction.sas +37 -0
- package/macros/mf_existvar.sas +39 -0
- package/macros/mf_existvarlist.sas +56 -0
- package/macros/mf_fmtdttm.sas +42 -0
- package/macros/mf_getapploc.sas +78 -0
- package/macros/mf_getattrc.sas +34 -0
- package/macros/mf_getattrn.sas +34 -0
- package/macros/mf_getengine.sas +54 -0
- package/macros/mf_getfilesize.sas +64 -0
- package/macros/mf_getfmtlist.sas +61 -0
- package/macros/mf_getfmtname.sas +44 -0
- package/macros/mf_getkeyvalue.sas +32 -0
- package/macros/mf_getplatform.sas +75 -0
- package/macros/mf_getquotedstr.sas +57 -0
- package/macros/mf_getschema.sas +43 -0
- package/macros/mf_getuniquefileref.sas +60 -0
- package/macros/mf_getuniquelibref.sas +57 -0
- package/macros/mf_getuniquename.sas +22 -0
- package/macros/mf_getuser.sas +44 -0
- package/macros/mf_getvalue.sas +36 -0
- package/macros/mf_getvarcount.sas +49 -0
- package/macros/mf_getvarformat.sas +72 -0
- package/macros/mf_getvarlen.sas +56 -0
- package/macros/mf_getvarlist.sas +74 -0
- package/macros/mf_getvarnum.sas +58 -0
- package/macros/mf_getvartype.sas +52 -0
- package/macros/mf_getxengine.sas +43 -0
- package/macros/mf_increment.sas +29 -0
- package/macros/mf_isblank.sas +27 -0
- package/macros/mf_isdir.sas +34 -0
- package/macros/mf_isint.sas +36 -0
- package/macros/mf_islibds.sas +40 -0
- package/macros/mf_loc.sas +29 -0
- package/macros/mf_mkdir.sas +67 -0
- package/macros/mf_mval.sas +19 -0
- package/macros/mf_nobs.sas +26 -0
- package/macros/mf_trimstr.sas +50 -0
- package/macros/mf_uid.sas +21 -0
- package/macros/mf_verifymacvars.sas +72 -0
- package/macros/mf_wordsinstr1andstr2.sas +53 -0
- package/macros/mf_wordsinstr1butnotstr2.sas +54 -0
- package/macros/mf_writefile.sas +67 -0
- package/macros/mp_abort.sas +301 -0
- package/macros/mp_appendfile.sas +57 -0
- package/macros/mp_applyformats.sas +181 -0
- package/macros/mp_assert.sas +56 -0
- package/macros/mp_assertcols.sas +145 -0
- package/macros/mp_assertcolvals.sas +168 -0
- package/macros/mp_assertdsobs.sas +121 -0
- package/macros/mp_assertscope.sas +147 -0
- package/macros/mp_base64copy.sas +117 -0
- package/macros/mp_binarycopy.sas +79 -0
- package/macros/mp_chop.sas +194 -0
- package/macros/mp_cleancsv.sas +78 -0
- package/macros/mp_cntlout.sas +85 -0
- package/macros/mp_copyfolder.sas +84 -0
- package/macros/mp_coretable.sas +73 -0
- package/macros/mp_createconstraints.sas +70 -0
- package/macros/mp_createwebservice.sas +32 -0
- package/macros/mp_csv2ds.sas +140 -0
- package/macros/mp_deleteconstraints.sas +52 -0
- package/macros/mp_deletefolder.sas +70 -0
- package/macros/mp_dictionary.sas +52 -0
- package/macros/mp_dirlist.sas +263 -0
- package/macros/mp_distinctfmtvalues.sas +50 -0
- package/macros/mp_dropmembers.sas +46 -0
- package/macros/mp_ds2cards.sas +291 -0
- package/macros/mp_ds2csv.sas +212 -0
- package/macros/mp_ds2ddl.sas +30 -0
- package/macros/mp_ds2fmtds.sas +105 -0
- package/macros/mp_ds2inserts.sas +180 -0
- package/macros/mp_ds2md.sas +103 -0
- package/macros/mp_ds2squeeze.sas +120 -0
- package/macros/mp_dsmeta.sas +110 -0
- package/macros/mp_filtercheck.sas +251 -0
- package/macros/mp_filtergenerate.sas +105 -0
- package/macros/mp_filterstore.sas +254 -0
- package/macros/mp_filtervalidate.sas +105 -0
- package/macros/mp_getcols.sas +67 -0
- package/macros/mp_getconstraints.sas +115 -0
- package/macros/mp_getdbml.sas +335 -0
- package/macros/mp_getddl.sas +400 -0
- package/macros/mp_getformats.sas +124 -0
- package/macros/mp_getmaxvarlengths.sas +130 -0
- package/macros/mp_getpk.sas +281 -0
- package/macros/mp_gsubfile.sas +58 -0
- package/macros/mp_guesspk.sas +332 -0
- package/macros/mp_hashdataset.sas +91 -0
- package/macros/mp_hashdirectory.sas +162 -0
- package/macros/mp_include.sas +106 -0
- package/macros/mp_init.sas +76 -0
- package/macros/mp_jsonout.sas +406 -0
- package/macros/mp_lib2cards.sas +78 -0
- package/macros/mp_lib2inserts.sas +77 -0
- package/macros/mp_loadformat.sas +303 -0
- package/macros/mp_lockanytable.sas +243 -0
- package/macros/mp_lockfilecheck.sas +103 -0
- package/macros/mp_makedata.sas +107 -0
- package/macros/mp_md5.sas +58 -0
- package/macros/mp_perflog.sas +43 -0
- package/macros/mp_prevobs.sas +88 -0
- package/macros/mp_recursivejoin.sas +90 -0
- package/macros/mp_replace.sas +151 -0
- package/macros/mp_reseterror.sas +27 -0
- package/macros/mp_resetoption.sas +39 -0
- package/macros/mp_retainedkey.sas +247 -0
- package/macros/mp_runddl.sas +49 -0
- package/macros/mp_searchcols.sas +103 -0
- package/macros/mp_searchdata.sas +131 -0
- package/macros/mp_setkeyvalue.sas +55 -0
- package/macros/mp_sortinplace.sas +122 -0
- package/macros/mp_stackdiffs.sas +592 -0
- package/macros/mp_storediffs.sas +230 -0
- package/macros/mp_stprequests.sas +74 -0
- package/macros/mp_streamfile.sas +212 -0
- package/macros/mp_testjob.sas +92 -0
- package/macros/mp_testservice.sas +39 -0
- package/macros/mp_testwritespeedlibrary.sas +59 -0
- package/macros/mp_tree.sas +71 -0
- package/macros/mp_unzip.sas +90 -0
- package/macros/mp_updatevarlength.sas +93 -0
- package/macros/mp_validatecol.sas +97 -0
- package/macros/mp_wait4file.sas +35 -0
- package/macros/mp_webin.sas +59 -0
- package/macros/mp_zip.sas +85 -0
- package/package.json +4 -1
- package/scripts/copySASjsCore.ts +36 -0
- package/utils/index.d.ts +1 -2
- package/utils/index.js +2 -3
- package/utils/index.js.map +1 -1
- package/utils/utils.d.ts +1 -0
- package/utils/utils.js +7 -1
- package/utils/utils.js.map +1 -1
- package/utils/getNodeModulePath.d.ts +0 -2
- package/utils/getNodeModulePath.js +0 -83
- package/utils/getNodeModulePath.js.map +0 -1
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file
|
|
3
|
+
@brief Converts deletes/changes/appends into a single audit table.
|
|
4
|
+
@details When tracking changes to data over time, it can be helpful to have
|
|
5
|
+
a single base table to track ALL modifications - enabling audit trail,
|
|
6
|
+
data recovery, and change re-application. This macro is one of many
|
|
7
|
+
data management utilities used in [Data Controller for SAS](
|
|
8
|
+
https:datacontroller.io) - a comprehensive data ingestion solution, which
|
|
9
|
+
works on any SAS platform (Viya, SAS 9, Foundation) and is free for up to 5
|
|
10
|
+
users.
|
|
11
|
+
|
|
12
|
+
NOTE - this macro does not validate the inputs. It is assumed that the
|
|
13
|
+
datasets containing the new / changed / deleted rows are CORRECT, contain
|
|
14
|
+
no additional (or missing columns), and that the originals dataset contains
|
|
15
|
+
all relevant base records (and no additionals).
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
|
|
19
|
+
data work.orig work.deleted work.changed work.appended;
|
|
20
|
+
set sashelp.class;
|
|
21
|
+
if _n_=1 then do;
|
|
22
|
+
output work.orig work.deleted;
|
|
23
|
+
end;
|
|
24
|
+
else if _n_=2 then do;
|
|
25
|
+
output work.orig;
|
|
26
|
+
age=99;
|
|
27
|
+
output work.changed;
|
|
28
|
+
end;
|
|
29
|
+
else do;
|
|
30
|
+
name='Newbie';
|
|
31
|
+
output work.appended;
|
|
32
|
+
stop;
|
|
33
|
+
end;
|
|
34
|
+
run;
|
|
35
|
+
|
|
36
|
+
%mp_storediffs(sashelp.class,work.orig,NAME
|
|
37
|
+
,delds=work.deleted
|
|
38
|
+
,modds=work.changed
|
|
39
|
+
,appds=work.appended
|
|
40
|
+
,outds=work.final
|
|
41
|
+
,mdebug=1
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
@param [in] libds Target table against which the changes were applied
|
|
45
|
+
@param [in] origds Dataset with original (unchanged) records. Can be empty if
|
|
46
|
+
only appending.
|
|
47
|
+
@param [in] key Space seperated list of key variables
|
|
48
|
+
@param [in] delds= (0) Dataset with deleted records
|
|
49
|
+
@param [in] appds= (0) Dataset with appended records
|
|
50
|
+
@param [in] modds= (0) Dataset with modified records
|
|
51
|
+
@param [out] outds= (work.mp_storediffs) Output table containing stored data.
|
|
52
|
+
DDL as follows: %mp_coretable(DIFFTABLE)
|
|
53
|
+
|
|
54
|
+
@param [in] processed_dttm= (0) Provide a datetime constant in relation to
|
|
55
|
+
the actual load time. If not provided, current timestamp is used.
|
|
56
|
+
@param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
|
|
57
|
+
@param [out] loadref= (0) Provide a unique key to reference the load,
|
|
58
|
+
otherwise a UUID will be generated.
|
|
59
|
+
|
|
60
|
+
<h4> SAS Macros </h4>
|
|
61
|
+
@li mf_getquotedstr.sas
|
|
62
|
+
@li mf_getuniquename.sas
|
|
63
|
+
@li mf_getvarlist.sas
|
|
64
|
+
|
|
65
|
+
<h4> Related Macros </h4>
|
|
66
|
+
@li mp_stackdiffs.sas
|
|
67
|
+
@li mp_storediffs.test.sas
|
|
68
|
+
|
|
69
|
+
@version 9.2
|
|
70
|
+
@author Allan Bowe
|
|
71
|
+
**/
|
|
72
|
+
/** @cond */
|
|
73
|
+
|
|
74
|
+
%macro mp_storediffs(libds
|
|
75
|
+
,origds
|
|
76
|
+
,key
|
|
77
|
+
,delds=0
|
|
78
|
+
,appds=0
|
|
79
|
+
,modds=0
|
|
80
|
+
,outds=work.mp_storediffs
|
|
81
|
+
,loadref=0
|
|
82
|
+
,processed_dttm=0
|
|
83
|
+
,mdebug=0
|
|
84
|
+
)/*/STORE SOURCE*/;
|
|
85
|
+
%local dbg;
|
|
86
|
+
%if &mdebug=1 %then %do;
|
|
87
|
+
%put &sysmacroname entry vars:;
|
|
88
|
+
%put _local_;
|
|
89
|
+
%end;
|
|
90
|
+
%else %let dbg=*;
|
|
91
|
+
|
|
92
|
+
/* set up unique and temporary vars */
|
|
93
|
+
%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
|
|
94
|
+
%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
|
|
95
|
+
%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
|
|
96
|
+
%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
|
|
97
|
+
%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_ds4));
|
|
98
|
+
%let hashkey=%upcase(%mf_getuniquename(prefix=mpsd_hashkey));
|
|
99
|
+
%let inds_auto=%upcase(%mf_getuniquename(prefix=mpsd_inds_auto));
|
|
100
|
+
%let inds_keep=%upcase(%mf_getuniquename(prefix=mpsd_inds_keep));
|
|
101
|
+
|
|
102
|
+
%let dslist=&origds;
|
|
103
|
+
%if &delds ne 0 %then %do;
|
|
104
|
+
%let delds=%upcase(&delds);
|
|
105
|
+
%if %scan(&delds,-1,.)=&delds %then %let delds=WORK.&delds;
|
|
106
|
+
%let dslist=&dslist &delds;
|
|
107
|
+
%end;
|
|
108
|
+
%if &appds ne 0 %then %do;
|
|
109
|
+
%let appds=%upcase(&appds);
|
|
110
|
+
%if %scan(&appds,-1,.)=&appds %then %let appds=WORK.&appds;
|
|
111
|
+
%let dslist=&dslist &appds;
|
|
112
|
+
%end;
|
|
113
|
+
%if &modds ne 0 %then %do;
|
|
114
|
+
%let modds=%upcase(&modds);
|
|
115
|
+
%if %scan(&modds,-1,.)=&modds %then %let modds=WORK.&modds;
|
|
116
|
+
%let dslist=&dslist &modds;
|
|
117
|
+
%end;
|
|
118
|
+
|
|
119
|
+
%let origds=%upcase(&origds);
|
|
120
|
+
%if %scan(&origds,-1,.)=&origds %then %let origds=WORK.&origds;
|
|
121
|
+
|
|
122
|
+
%let key=%upcase(&key);
|
|
123
|
+
|
|
124
|
+
/* hash the key and append all the tables (marking the source) */
|
|
125
|
+
data &ds1;
|
|
126
|
+
set &dslist indsname=&inds_auto;
|
|
127
|
+
&hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
|
|
128
|
+
&inds_keep=upcase(&inds_auto);
|
|
129
|
+
proc sort;
|
|
130
|
+
by &inds_keep &hashkey;
|
|
131
|
+
run;
|
|
132
|
+
|
|
133
|
+
/* transpose numeric & char vars */
|
|
134
|
+
proc transpose data=&ds1
|
|
135
|
+
out=&ds2(rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_num));
|
|
136
|
+
by &inds_keep &hashkey;
|
|
137
|
+
var _numeric_;
|
|
138
|
+
run;
|
|
139
|
+
proc transpose data=&ds1
|
|
140
|
+
out=&ds3(
|
|
141
|
+
rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_char)
|
|
142
|
+
where=(tgtvar_nm not in ("&hashkey","&inds_keep"))
|
|
143
|
+
);
|
|
144
|
+
by &inds_keep &hashkey;
|
|
145
|
+
var _character_;
|
|
146
|
+
run;
|
|
147
|
+
|
|
148
|
+
%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
|
|
149
|
+
/* this is a format catalog - cannot query cols directly */
|
|
150
|
+
%let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
|
|
151
|
+
,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
|
|
152
|
+
,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
|
|
153
|
+
%end;
|
|
154
|
+
%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
|
|
155
|
+
|
|
156
|
+
data &ds4;
|
|
157
|
+
length &inds_keep $41 tgtvar_nm $32 _label_ $256;
|
|
158
|
+
if _n_=1 then call missing(_label_);
|
|
159
|
+
drop _label_;
|
|
160
|
+
set &ds2 &ds3 indsname=&inds_auto;
|
|
161
|
+
|
|
162
|
+
tgtvar_nm=upcase(tgtvar_nm);
|
|
163
|
+
if tgtvar_nm in (%upcase(&vlist));
|
|
164
|
+
|
|
165
|
+
if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
|
|
166
|
+
else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
|
|
167
|
+
else do;
|
|
168
|
+
putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
|
|
169
|
+
call symputx('syscc',98);
|
|
170
|
+
end;
|
|
171
|
+
|
|
172
|
+
if &inds_keep="&appds" then move_type='A';
|
|
173
|
+
else if &inds_keep="&delds" then move_type='D';
|
|
174
|
+
else if &inds_keep="&modds" then move_type='M';
|
|
175
|
+
else if &inds_keep="&origds" then move_type='O';
|
|
176
|
+
else do;
|
|
177
|
+
putlog "%str(ERR)OR: unidentified movetype input!" &inds_keep;
|
|
178
|
+
call symputx('syscc',99);
|
|
179
|
+
end;
|
|
180
|
+
tgtvar_nm=upcase(tgtvar_nm);
|
|
181
|
+
if tgtvar_nm in (%mf_getquotedstr(&key)) then is_pk=1;
|
|
182
|
+
else is_pk=0;
|
|
183
|
+
drop &inds_keep;
|
|
184
|
+
run;
|
|
185
|
+
|
|
186
|
+
%if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
|
|
187
|
+
%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
|
|
188
|
+
%let libds=%upcase(&libds);
|
|
189
|
+
|
|
190
|
+
/* join orig vals for modified & deleted */
|
|
191
|
+
proc sql;
|
|
192
|
+
create table &outds as
|
|
193
|
+
select "&loadref" as load_ref length=36
|
|
194
|
+
,&processed_dttm as processed_dttm format=E8601DT26.6
|
|
195
|
+
,"%scan(&libds,1,.)" as libref length=8
|
|
196
|
+
,"%scan(&libds,2,.)" as dsn length=32
|
|
197
|
+
,b.key_hash length=32
|
|
198
|
+
,b.move_type length=1
|
|
199
|
+
,b.tgtvar_nm length=32
|
|
200
|
+
,b.is_pk
|
|
201
|
+
,case when b.move_type ne 'M' then -1
|
|
202
|
+
when a.newval_num=b.newval_num and a.newval_char=b.newval_char then 0
|
|
203
|
+
else 1
|
|
204
|
+
end as is_diff
|
|
205
|
+
,b.tgtvar_type length=1
|
|
206
|
+
,case when b.move_type='D' then b.newval_num
|
|
207
|
+
else a.newval_num
|
|
208
|
+
end as oldval_num format=best32.
|
|
209
|
+
,case when b.move_type='D' then .
|
|
210
|
+
else b.newval_num
|
|
211
|
+
end as newval_num format=best32.
|
|
212
|
+
,case when b.move_type='D' then b.newval_char
|
|
213
|
+
else a.newval_char
|
|
214
|
+
end as oldval_char length=32765
|
|
215
|
+
,case when b.move_type='D' then ''
|
|
216
|
+
else b.newval_char
|
|
217
|
+
end as newval_char length=32765
|
|
218
|
+
from &ds4(where=(move_type='O')) as a
|
|
219
|
+
right join &ds4(where=(move_type ne 'O')) as b
|
|
220
|
+
on a.tgtvar_nm=b.tgtvar_nm
|
|
221
|
+
and a.key_hash=b.key_hash
|
|
222
|
+
order by move_type, key_hash,is_pk desc, tgtvar_nm;
|
|
223
|
+
|
|
224
|
+
%if &mdebug=0 %then %do;
|
|
225
|
+
proc sql;
|
|
226
|
+
drop table &ds1, &ds2, &ds3, &ds4;
|
|
227
|
+
%end;
|
|
228
|
+
|
|
229
|
+
%mend mp_storediffs;
|
|
230
|
+
/** @endcond */
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file
|
|
3
|
+
@brief Capture session start / finish times and request details
|
|
4
|
+
@details For details, see
|
|
5
|
+
https://rawsas.com/event-logging-of-stored-process-server-sessions.
|
|
6
|
+
Requires a base table in the following structure (name can be changed):
|
|
7
|
+
|
|
8
|
+
proc sql;
|
|
9
|
+
create table &libds(
|
|
10
|
+
request_dttm num not null format=datetime.
|
|
11
|
+
,status_cd char(4) not null
|
|
12
|
+
,_metaperson varchar(100) not null
|
|
13
|
+
,_program varchar(500)
|
|
14
|
+
,sysuserid varchar(50)
|
|
15
|
+
,sysjobid varchar(12)
|
|
16
|
+
,_sessionid varchar(50)
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
Called via STP init / term events (configurable in SMC) as follows:
|
|
20
|
+
|
|
21
|
+
%mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET )
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@param status_cd= Use INIT for INIT and TERM for TERM events
|
|
25
|
+
@param libds= Location of base table (library.dataset). To minimise risk
|
|
26
|
+
of table locks, we HIGHLY recommend using a database (NOT a SAS dataset).
|
|
27
|
+
THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the
|
|
28
|
+
init program proper.
|
|
29
|
+
|
|
30
|
+
@version 9.2
|
|
31
|
+
@author Allan Bowe
|
|
32
|
+
@source https://github.com/sasjs/core
|
|
33
|
+
|
|
34
|
+
**/
|
|
35
|
+
|
|
36
|
+
%macro mp_stprequests(status_cd= /* $4 eg INIT or TERM */
|
|
37
|
+
,libds=somelib.stp_requests /* base table location */
|
|
38
|
+
)/*/STORE SOURCE*/;
|
|
39
|
+
|
|
40
|
+
/* set nosyntaxcheck so the code runs regardless */
|
|
41
|
+
%local etls_syntaxcheck;
|
|
42
|
+
%let etls_syntaxcheck=%sysfunc(getoption(syntaxcheck));
|
|
43
|
+
options nosyntaxcheck;
|
|
44
|
+
|
|
45
|
+
data ;
|
|
46
|
+
if 0 then set &libds;
|
|
47
|
+
request_dttm=datetime();
|
|
48
|
+
status_cd="&status_cd";
|
|
49
|
+
_METAPERSON="&_metaperson";
|
|
50
|
+
_PROGRAM="&_program";
|
|
51
|
+
SYSUSERID="&sysuserid";
|
|
52
|
+
SYSJOBID="&sysjobid";
|
|
53
|
+
%if not %symexist(_SESSIONID) %then %do;
|
|
54
|
+
/* session id is stored in the replay variable but needs to be extracted */
|
|
55
|
+
_replay=symget('_replay');
|
|
56
|
+
_replay=subpad(_replay,index(_replay,'_sessionid=')+11,length(_replay));
|
|
57
|
+
index=index(_replay,'&')-1;
|
|
58
|
+
if index=-1 then index=length(_replay);
|
|
59
|
+
_replay=substr(_replay,1,index);
|
|
60
|
+
_SESSIONID=_replay;
|
|
61
|
+
drop _replay index;
|
|
62
|
+
%end;
|
|
63
|
+
%else %do;
|
|
64
|
+
/* explicitly created sessions are automatically available */
|
|
65
|
+
_SESSIONID=symget('_SESSIONID');
|
|
66
|
+
%end;
|
|
67
|
+
output;
|
|
68
|
+
stop;
|
|
69
|
+
run;
|
|
70
|
+
|
|
71
|
+
proc append base=&libds data=&syslast nowarn;run;
|
|
72
|
+
|
|
73
|
+
options &etls_syntaxcheck;
|
|
74
|
+
%mend mp_stprequests;
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file
|
|
3
|
+
@brief Streams a file to _webout according to content type
|
|
4
|
+
@details Will set headers using appropriate functions per the server type
|
|
5
|
+
(Viya, EBI, [SASjs Server](https://github.com/sasjs/server)) and stream
|
|
6
|
+
content using mp_binarycopy().
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
|
|
10
|
+
filename mc url
|
|
11
|
+
"https://raw.githubusercontent.com/sasjs/core/main/all.sas";
|
|
12
|
+
%inc mc;
|
|
13
|
+
|
|
14
|
+
%mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
|
|
15
|
+
|
|
16
|
+
@param [in] contenttype= (TEXT) Supported:
|
|
17
|
+
@li CSV
|
|
18
|
+
@li EXCEL
|
|
19
|
+
@li MARKDOWN
|
|
20
|
+
@li TEXT
|
|
21
|
+
@li ZIP
|
|
22
|
+
Feel free to submit PRs to support more mime types! The official list is
|
|
23
|
+
here: https://www.iana.org/assignments/media-types/media-types.xhtml
|
|
24
|
+
@param [in] inloc= /path/to/file.ext to be sent
|
|
25
|
+
@param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
|
|
26
|
+
@param [in] iftrue= (1=1) Provide a condition under which to execute.
|
|
27
|
+
@param [out] outname= the name of the file, as downloaded by the browser
|
|
28
|
+
@param [out] outref= (_webout) The destination where the file should be
|
|
29
|
+
streamed.
|
|
30
|
+
|
|
31
|
+
<h4> SAS Macros </h4>
|
|
32
|
+
@li mf_getplatform.sas
|
|
33
|
+
@li mfs_httpheader.sas
|
|
34
|
+
@li mp_binarycopy.sas
|
|
35
|
+
|
|
36
|
+
@author Allan Bowe
|
|
37
|
+
|
|
38
|
+
**/
|
|
39
|
+
|
|
40
|
+
%macro mp_streamfile(
|
|
41
|
+
contenttype=TEXT
|
|
42
|
+
,inloc=
|
|
43
|
+
,inref=0
|
|
44
|
+
,iftrue=%str(1=1)
|
|
45
|
+
,outname=
|
|
46
|
+
,outref=_webout
|
|
47
|
+
)/*/STORE SOURCE*/;
|
|
48
|
+
|
|
49
|
+
%if not(%eval(%unquote(&iftrue))) %then %return;
|
|
50
|
+
|
|
51
|
+
%let contentype=%upcase(&contenttype);
|
|
52
|
+
%let outref=%upcase(&outref);
|
|
53
|
+
%local platform; %let platform=%mf_getplatform();
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* check engine type to avoid the below err message:
|
|
57
|
+
* > Function is only valid for filerefs using the CACHE access method.
|
|
58
|
+
*/
|
|
59
|
+
%local streamweb;
|
|
60
|
+
%let streamweb=0;
|
|
61
|
+
data _null_;
|
|
62
|
+
set sashelp.vextfl(where=(upcase(fileref)="&outref"));
|
|
63
|
+
if xengine='STREAM' then call symputx('streamweb',1,'l');
|
|
64
|
+
run;
|
|
65
|
+
|
|
66
|
+
%if &contentype=CSV %then %do;
|
|
67
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
68
|
+
data _null_;
|
|
69
|
+
rc=stpsrv_header('Content-Type','application/csv');
|
|
70
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
71
|
+
run;
|
|
72
|
+
%end;
|
|
73
|
+
%else %if &platform=SASVIYA %then %do;
|
|
74
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
|
|
75
|
+
contenttype='application/csv'
|
|
76
|
+
contentdisp="attachment; filename=&outname";
|
|
77
|
+
%end;
|
|
78
|
+
%else %if &platform=SASJS %then %do;
|
|
79
|
+
%mfs_httpheader(Content-Type,application/csv)
|
|
80
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
81
|
+
%end;
|
|
82
|
+
%end;
|
|
83
|
+
%else %if &contentype=EXCEL %then %do;
|
|
84
|
+
/* suitable for XLS format */
|
|
85
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
86
|
+
data _null_;
|
|
87
|
+
rc=stpsrv_header('Content-Type','application/vnd.ms-excel');
|
|
88
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
89
|
+
run;
|
|
90
|
+
%end;
|
|
91
|
+
%else %if &platform=SASVIYA %then %do;
|
|
92
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
|
|
93
|
+
contenttype='application/vnd.ms-excel'
|
|
94
|
+
contentdisp="attachment; filename=&outname";
|
|
95
|
+
%end;
|
|
96
|
+
%else %if &platform=SASJS %then %do;
|
|
97
|
+
%mfs_httpheader(Content-Type,application/vnd.ms-excel)
|
|
98
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
99
|
+
%end;
|
|
100
|
+
%end;
|
|
101
|
+
%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
|
|
102
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
103
|
+
data _null_;
|
|
104
|
+
rc=stpsrv_header('Content-Type',"image/%lowcase(&contenttype)");
|
|
105
|
+
run;
|
|
106
|
+
%end;
|
|
107
|
+
%else %if &platform=SASVIYA %then %do;
|
|
108
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
|
|
109
|
+
contenttype="image/%lowcase(&contenttype)";
|
|
110
|
+
%end;
|
|
111
|
+
%else %if &platform=SASJS %then %do;
|
|
112
|
+
%mfs_httpheader(Content-Type,image/%lowcase(&contenttype))
|
|
113
|
+
%end;
|
|
114
|
+
%end;
|
|
115
|
+
%else %if &contentype=HTML or &contenttype=MARKDOWN %then %do;
|
|
116
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
117
|
+
data _null_;
|
|
118
|
+
rc=stpsrv_header('Content-Type',"text/%lowcase(&contenttype)");
|
|
119
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
120
|
+
run;
|
|
121
|
+
%end;
|
|
122
|
+
%else %if &platform=SASVIYA %then %do;
|
|
123
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
|
|
124
|
+
contenttype="text/%lowcase(&contenttype)"
|
|
125
|
+
contentdisp="attachment; filename=&outname";
|
|
126
|
+
%end;
|
|
127
|
+
%else %if &platform=SASJS %then %do;
|
|
128
|
+
%mfs_httpheader(Content-Type,text/%lowcase(&contenttype))
|
|
129
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
130
|
+
%end;
|
|
131
|
+
%end;
|
|
132
|
+
%else %if &contentype=TEXT %then %do;
|
|
133
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
134
|
+
data _null_;
|
|
135
|
+
rc=stpsrv_header('Content-Type','application/text');
|
|
136
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
137
|
+
run;
|
|
138
|
+
%end;
|
|
139
|
+
%else %if &platform=SASVIYA %then %do;
|
|
140
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt'
|
|
141
|
+
contenttype='application/text'
|
|
142
|
+
contentdisp="attachment; filename=&outname";
|
|
143
|
+
%end;
|
|
144
|
+
%else %if &platform=SASJS %then %do;
|
|
145
|
+
%mfs_httpheader(Content-Type,application/text)
|
|
146
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
147
|
+
%end;
|
|
148
|
+
%end;
|
|
149
|
+
%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
|
|
150
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
151
|
+
data _null_;
|
|
152
|
+
rc=stpsrv_header('Content-Type',"font/%lowcase(&contenttype)");
|
|
153
|
+
run;
|
|
154
|
+
%end;
|
|
155
|
+
%else %if &platform=SASVIYA %then %do;
|
|
156
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
|
|
157
|
+
contenttype="font/%lowcase(&contenttype)";
|
|
158
|
+
%end;
|
|
159
|
+
%else %if &platform=SASJS %then %do;
|
|
160
|
+
%mfs_httpheader(Content-Type,font/%lowcase(&contenttype))
|
|
161
|
+
%end;
|
|
162
|
+
%end;
|
|
163
|
+
%else %if &contentype=XLSX %then %do;
|
|
164
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
165
|
+
data _null_;
|
|
166
|
+
rc=stpsrv_header('Content-Type',
|
|
167
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
|
|
168
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
169
|
+
run;
|
|
170
|
+
%end;
|
|
171
|
+
%else %if &platform=SASVIYA %then %do;
|
|
172
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
|
|
173
|
+
contenttype=
|
|
174
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
|
|
175
|
+
contentdisp="attachment; filename=&outname";
|
|
176
|
+
%end;
|
|
177
|
+
%else %if &platform=SASJS %then %do;
|
|
178
|
+
%mfs_httpheader(Content-Type
|
|
179
|
+
,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
|
|
180
|
+
)
|
|
181
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
182
|
+
%end;
|
|
183
|
+
%end;
|
|
184
|
+
%else %if &contentype=ZIP %then %do;
|
|
185
|
+
%if (&platform=SASMETA and &streamweb=1) %then %do;
|
|
186
|
+
data _null_;
|
|
187
|
+
rc=stpsrv_header('Content-Type','application/zip');
|
|
188
|
+
rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
|
|
189
|
+
run;
|
|
190
|
+
%end;
|
|
191
|
+
%else %if &platform=SASVIYA %then %do;
|
|
192
|
+
filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
|
|
193
|
+
contenttype='application/zip'
|
|
194
|
+
contentdisp="attachment; filename=&outname";
|
|
195
|
+
%end;
|
|
196
|
+
%else %if &platform=SASJS %then %do;
|
|
197
|
+
%mfs_httpheader(Content-Type,application/zip)
|
|
198
|
+
%mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
|
|
199
|
+
%end;
|
|
200
|
+
%end;
|
|
201
|
+
%else %do;
|
|
202
|
+
%put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;
|
|
203
|
+
%end;
|
|
204
|
+
|
|
205
|
+
%if &inref ne 0 %then %do;
|
|
206
|
+
%mp_binarycopy(inref=&inref,outref=&outref)
|
|
207
|
+
%end;
|
|
208
|
+
%else %do;
|
|
209
|
+
%mp_binarycopy(inloc="&inloc",outref=&outref)
|
|
210
|
+
%end;
|
|
211
|
+
|
|
212
|
+
%mend mp_streamfile;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file
|
|
3
|
+
@brief Runs arbitrary code for a specified amount of time
|
|
4
|
+
@details Executes a series of procs and data steps to enable performance
|
|
5
|
+
testing of arbitrary jobs.
|
|
6
|
+
|
|
7
|
+
%mp_testjob(
|
|
8
|
+
duration=60*5
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
@param [in] duration= the time in seconds which the job should run for. Actual
|
|
12
|
+
time may vary, as the check is done in between steps. Default = 30 (seconds).
|
|
13
|
+
|
|
14
|
+
<h4> SAS Macros </h4>
|
|
15
|
+
@li mf_getuniquelibref.sas
|
|
16
|
+
@li mf_getuniquename.sas
|
|
17
|
+
@li mf_mkdir.sas
|
|
18
|
+
|
|
19
|
+
@version 9.4
|
|
20
|
+
@author Allan Bowe
|
|
21
|
+
|
|
22
|
+
**/
|
|
23
|
+
|
|
24
|
+
%macro mp_testjob(duration=30
|
|
25
|
+
)/*/STORE SOURCE*/;
|
|
26
|
+
%local lib dir ds1 ds2 ds3 start_tm i;
|
|
27
|
+
|
|
28
|
+
%let start_tm=%sysfunc(datetime());
|
|
29
|
+
%let duration=%sysevalf(&duration);
|
|
30
|
+
|
|
31
|
+
/* create a temporary library in WORK */
|
|
32
|
+
%let lib=%mf_getuniquelibref();
|
|
33
|
+
%let dir=%mf_getuniquename();
|
|
34
|
+
%mf_mkdir(%sysfunc(pathname(work))/&dir)
|
|
35
|
+
libname &lib "%sysfunc(pathname(work))/&dir";
|
|
36
|
+
|
|
37
|
+
/* loop through until time expires */
|
|
38
|
+
%let ds1=%mf_getuniquename();
|
|
39
|
+
%let ds2=%mf_getuniquename();
|
|
40
|
+
%let ds3=%mf_getuniquename();
|
|
41
|
+
%do i=0 %to 1;
|
|
42
|
+
|
|
43
|
+
/* create big dataset */
|
|
44
|
+
data &lib..&ds1(compress=no );
|
|
45
|
+
do x=1 to 1000000;
|
|
46
|
+
randnum0=ranuni(0)*3;
|
|
47
|
+
randnum1=ranuni(0)*2;
|
|
48
|
+
bigchar=repeat('A',300);
|
|
49
|
+
output;
|
|
50
|
+
end;
|
|
51
|
+
run;
|
|
52
|
+
%if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
|
|
53
|
+
|
|
54
|
+
proc summary ;
|
|
55
|
+
class randnum0 randnum1;
|
|
56
|
+
output out=&lib..&ds2;
|
|
57
|
+
run;quit;
|
|
58
|
+
%if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
|
|
59
|
+
|
|
60
|
+
/* add more data */
|
|
61
|
+
proc sql;
|
|
62
|
+
create table &lib..&ds3 as
|
|
63
|
+
select *, ranuni(0)*10 as randnum2
|
|
64
|
+
from &lib..&ds1
|
|
65
|
+
order by randnum1;
|
|
66
|
+
quit;
|
|
67
|
+
%if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
|
|
68
|
+
|
|
69
|
+
proc sort data=&lib..&ds3;
|
|
70
|
+
by descending x;
|
|
71
|
+
run;
|
|
72
|
+
%if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
|
|
73
|
+
|
|
74
|
+
/* wait 5 seconds */
|
|
75
|
+
data _null_;
|
|
76
|
+
call sleep(5,1);
|
|
77
|
+
run;
|
|
78
|
+
%if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate;
|
|
79
|
+
|
|
80
|
+
%let i=0;
|
|
81
|
+
|
|
82
|
+
%end;
|
|
83
|
+
|
|
84
|
+
%gate:
|
|
85
|
+
%put time is up!;
|
|
86
|
+
proc datasets lib=&lib kill;
|
|
87
|
+
run;
|
|
88
|
+
quit;
|
|
89
|
+
libname &lib clear;
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
%mend mp_testjob;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file
|
|
3
|
+
@brief To be deprecated. Will execute a SASjs web service on SAS 9 or Viya
|
|
4
|
+
@details Use the mx_testservice.sas macro instead (documentation can be
|
|
5
|
+
found there)
|
|
6
|
+
|
|
7
|
+
<h4> SAS Macros </h4>
|
|
8
|
+
@li mx_testservice.sas
|
|
9
|
+
|
|
10
|
+
@version 9.4
|
|
11
|
+
@author Allan Bowe
|
|
12
|
+
|
|
13
|
+
**/
|
|
14
|
+
|
|
15
|
+
%macro mp_testservice(program,
|
|
16
|
+
inputfiles=0,
|
|
17
|
+
inputdatasets=0,
|
|
18
|
+
inputparams=0,
|
|
19
|
+
debug=log,
|
|
20
|
+
mdebug=0,
|
|
21
|
+
outlib=0,
|
|
22
|
+
outref=0,
|
|
23
|
+
viyaresult=WEBOUT_JSON,
|
|
24
|
+
viyacontext=SAS Job Execution compute context
|
|
25
|
+
)/*/STORE SOURCE*/;
|
|
26
|
+
|
|
27
|
+
%mx_testservice(&program,
|
|
28
|
+
inputfiles=&inputfiles,
|
|
29
|
+
inputdatasets=&inputdatasets,
|
|
30
|
+
inputparams=&inputparams,
|
|
31
|
+
debug=&debug,
|
|
32
|
+
mdebug=&mdebug,
|
|
33
|
+
outlib=&outlib,
|
|
34
|
+
outref=&outref,
|
|
35
|
+
viyaresult=&viyaresult,
|
|
36
|
+
viyacontext=&viyacontext
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
%mend mp_testservice;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
@file mp_testwritespeedlibrary.sas
|
|
3
|
+
@brief Tests the write speed of a new table in a SAS library
|
|
4
|
+
@details Will create a new table of a certain size in an
|
|
5
|
+
existing SAS library. The table will have one column,
|
|
6
|
+
and will be subsequently deleted.
|
|
7
|
+
|
|
8
|
+
%mp_testwritespeedlibrary(
|
|
9
|
+
lib=work
|
|
10
|
+
,size=0.5
|
|
11
|
+
,outds=work.results
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
@param lib= (WORK) The library in which to create the table
|
|
15
|
+
@param size= (0.1) The size in GB of the table to create
|
|
16
|
+
@param outds= (WORK.RESULTS) The output dataset to be created.
|
|
17
|
+
|
|
18
|
+
<h4> SAS Macros </h4>
|
|
19
|
+
@li mf_getuniquename.sas
|
|
20
|
+
@li mf_existds.sas
|
|
21
|
+
|
|
22
|
+
@version 9.4
|
|
23
|
+
@author Allan Bowe
|
|
24
|
+
|
|
25
|
+
**/
|
|
26
|
+
|
|
27
|
+
%macro mp_testwritespeedlibrary(lib=WORK
|
|
28
|
+
,outds=work.results
|
|
29
|
+
,size=0.1
|
|
30
|
+
)/*/STORE SOURCE*/;
|
|
31
|
+
%local ds start;
|
|
32
|
+
|
|
33
|
+
/* find an unused, unique name for the new table */
|
|
34
|
+
%let ds=%mf_getuniquename();
|
|
35
|
+
%do %until(%mf_existds(&lib..&ds)=0);
|
|
36
|
+
%let ds=%mf_getuniquename();
|
|
37
|
+
%end;
|
|
38
|
+
|
|
39
|
+
%let start=%sysfunc(datetime());
|
|
40
|
+
|
|
41
|
+
data &lib..&ds(compress=no keep=x);
|
|
42
|
+
header=128*1024;
|
|
43
|
+
size=(1073741824/8 * &size) - header;
|
|
44
|
+
do x=1 to size;
|
|
45
|
+
output;
|
|
46
|
+
end;
|
|
47
|
+
run;
|
|
48
|
+
|
|
49
|
+
proc sql;
|
|
50
|
+
drop table &lib..&ds;
|
|
51
|
+
|
|
52
|
+
data &outds;
|
|
53
|
+
lib="&lib";
|
|
54
|
+
start_dttm=put(&start,datetime19.);
|
|
55
|
+
end_dttm=put(datetime(),datetime19.);
|
|
56
|
+
duration_seconds=end_dttm-start_dttm;
|
|
57
|
+
run;
|
|
58
|
+
|
|
59
|
+
%mend mp_testwritespeedlibrary;
|