@sasjs/utils 2.49.0 → 2.49.2

package/macros/mp_assertdsobs.sas

@@ -0,0 +1,121 @@
+/**
+  @file
+  @brief Asserts the number of observations in a dataset
+  @details Useful in the context of writing sasjs tests.  The results of the
+  test are _appended_ to the &outds. table.
+
+  Example usage:
+
+      %mp_assertdsobs(sashelp.class) %* tests if any observations are present;
+
+      %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present;
+
+      %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present;
+
+
+  @param [in] inds input dataset to test for presence of observations
+  @param [in] desc= (Testing observations) The user provided test description
+  @param [in] test= (HASOBS) The test to apply.  Valid values are:
+    @li HASOBS - Test is a PASS if the input dataset has any observations
+    @li EMPTY - Test is a PASS if input dataset is empty
+    @li EQUALS [integer] - Test passes if row count matches the provided integer
+    @li ATLEAST [integer] - Test passes if row count is more than or equal to
+      the provided integer
+    @li ATMOST [integer] - Test passes if row count is less than or equal to
+      the provided integer
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|Dataset &inds has XX obs|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assertcolvals.sas
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertdsobs(inds,
+  test=HASOBS,
+  desc=Testing observations,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+
+  %local nobs ds;
+  %let nobs=%mf_nobs(&inds);
+  %let test=%upcase(&test);
+  %let ds=%mf_getuniquename(prefix=mp_assertdsobs);
+
+  %if %substr(&test.xxxxx,1,6)=EQUALS %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected EQUALS [integer])
+    )
+    %let test=EQUALS;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATLEAST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATLEAST [integer])
+    )
+    %let test=ATLEAST;
+  %end;
+  %else %if %substr(&test.xxxxxxx,1,7)=ATMOST %then %do;
+    %let val=%scan(&test,2,%str( ));
+    %mp_abort(iftrue= (%DATATYP(&val)=CHAR)
+      ,mac=&sysmacroname
+      ,msg=%str(Invalid test - &test, expected ATMOST [integer])
+    )
+    %let test=ATMOST;
+  %end;
+  %else %if &test ne HASOBS and &test ne EMPTY %then %do;
+    %mp_abort(
+      mac=&sysmacroname,
+      msg=%str(Invalid test - &test)
+    )
+  %end;
+
+  data &ds;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_result='FAIL';
+    test_comments="&sysmacroname: Dataset &inds has &nobs observations.";
+    test_comments=test_comments!!" Test was "!!symget('test');
+  %if &test=HASOBS %then %do;
+    if &nobs>0 then test_result='PASS';
+  %end;
+  %else %if &test=EMPTY %then %do;
+    if &nobs=0 then test_result='PASS';
+  %end;
+  %else %if &test=EQUALS %then %do;
+    if &nobs=&val then test_result='PASS';
+  %end;
+  %else %if &test=ATLEAST %then %do;
+    if &nobs ge &val then test_result='PASS';
+  %end;
+  %else %if &test=ATMOST %then %do;
+    if &nobs le &val then test_result='PASS';
+  %end;
+  %else %do;
+    test_comments="&sysmacroname: Unsatisfied test condition - &test";
+  %end;
+  run;
+
+  proc append base=&outds data=&ds;
+  run;
+
+  proc sql;
+  drop table &ds;
+
+%mend mp_assertdsobs;

package/macros/mp_assertscope.sas

@@ -0,0 +1,147 @@
+/**
+  @file
+  @brief Used to capture scope leakage of macro variables
+  @details
+
+  A common 'difficult to detect' bug in macros is where a nested macro
+  over-writes variables in a higher level macro.
+
+  This assertion takes a snapshot of the macro variables before and after
+  a macro invocation.  Differences are captured in the `&outds` table. This
+  makes it easy to detect whether any macro variables were modified or
+  changed.
+
+  The following variables are NOT tested (as they are known, global variables
+  used in SASjs):
+
+  @li &sasjs_prefix._FUNCTIONS
+
+  Global variables are initialised in mp_init.sas - which will also trigger
+  "strict mode" in your SAS session.  Whilst this is a default in SASjs
+  produced apps, if you prefer not to use this mode, simply instantiate the
+  following variable to prevent the macro from running:  `SASJS_PREFIX`
+
+  Example usage:
+
+      %mp_assertscope(SNAPSHOT)
+
+      %let oops=I did it again;
+
+      %mp_assertscope(COMPARE,
+        desc=Checking macro variables against previous snapshot
+      )
+
+  This macro is designed to work alongside `sasjs test` - for more information
+  about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test).
+
+  @param [in] action (SNAPSHOT) The action to take.  Valid values:
+    @li SNAPSHOT - take a copy of the current macro variables
+    @li COMPARE - compare the current macro variables against previous values
+  @param [in] scope= (GLOBAL) The scope of the variables to be checked.  This
+    corresponds to the values in the SCOPE column in `sashelp.vmacro`.
+  @param [in] desc= (Testing scope leakage) The user provided test description
+  @param [in] ignorelist= Provide a list of macro variable names to ignore from
+    the comparison
+  @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the
+    scope snapshot
+  @param [out] outds= (work.test_results) The output dataset to contain the
+  results.  If it does not exist, it will be created, with the following format:
+  |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256|
+  |---|---|---|
+  |User Provided description|PASS|No out of scope variables created or modified|
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_init.sas
+
+  <h4> Related Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertcols.sas
+  @li mp_assertcolvals.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_assertscope(action,
+  desc=Testing Scope Leakage,
+  scope=GLOBAL,
+  scopeds=work.mp_assertscope,
+  ignorelist=,
+  outds=work.test_results
+)/*/STORE SOURCE*/;
+%local ds test_result test_comments del add mod ilist;
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
+
+/**
+  * this sets up the global vars, it will also enter STRICT mode.  If this
+  * behaviour is not desired, simply initiate the following global macro
+  * variable to prevent the macro from running: SASJS_PREFIX
+  */
+%mp_init()
+
+/* get current variables */
+%if &action=SNAPSHOT %then %do;
+  proc sql;
+  create table &scopeds as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+%end;
+%else %if &action=COMPARE %then %do;
+
+  proc sql;
+  create table _data_ as
+    select name,offset,value
+    from dictionary.macros
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
+    order by name,offset;
+
+  %let ds=&syslast;
+
+  proc compare
+    base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
+    compare=&ds noprint;
+  run;
+
+  %if &sysinfo=0 %then %do;
+    %let test_result=PASS;
+    %let test_comments=&scope Variables Unmodified;
+  %end;
+  %else %do;
+    proc sql noprint undo_policy=none;
+    select distinct name into: del separated by ' '  from &scopeds
+      where name not in (select name from &ds);
+    select distinct name into: add separated by ' '  from &ds
+      where name not in (select name from &scopeds);
+    select distinct a.name into: mod separated by ' '
+      from &scopeds a
+      inner join &ds b
+      on a.name=b.name
+        and a.offset=b.offset
+      where a.value ne b.value;
+    %let test_result=FAIL;
+    %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
+  %end;
+
+
+  data ;
+    length test_description $256 test_result $4 test_comments $256;
+    test_description=symget('desc');
+    test_comments=symget('test_comments');
+    test_result=symget('test_result');
+  run;
+
+  %let ds=&syslast;
+  proc append base=&outds data=&ds;
+  run;
+  proc sql;
+  drop table &ds;
+%end;
+
+%mend mp_assertscope;

package/macros/mp_base64copy.sas

@@ -0,0 +1,117 @@
+/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;

package/macros/mp_binarycopy.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief Copy any file using binary input / output streams
+  @details Reads in a file byte by byte and writes it back out.  Is an
+  os-independent method to copy files.  In case of naming collision, the
+  default filerefs can be modified.
+  Note that if you have a new enough version of SAS, and you don't need features
+  such as APPEND, you may be better of using the fcopy() function instead.
+
+        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
+
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+
+  @returns nothing
+
+  @version 9.2
+
+**/
+
+%macro mp_binarycopy(
+    inloc=           /* full path and filename of the object to be copied */
+    ,outloc=          /* full path and filename of object to be created */
+    ,inref=____in   /* override default to use own filerefs */
+    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
+    ,iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+  %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if &mode=APPEND %then %let mod=mod;
+
+  /* these IN and OUT filerefs can point to anything */
+  %if &inref = ____in %then %do;
+    filename &inref &inloc lrecl=1048576 ;
+  %end;
+  %if &outref=____out %then %do;
+    filename &outref &outloc lrecl=1048576 &mod;
+  %end;
+
+  /* copy the file byte-for-byte  */
+  data _null_;
+    infile &inref lrecl=1 recfm=n;
+    file &outref &mod recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+
+  %if &inref = ____in %then %do;
+    filename &inref clear;
+  %end;
+  %if &outref=____out %then %do;
+    filename &outref clear;
+  %end;
+%mend mp_binarycopy;

package/macros/mp_chop.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief Splits a file of ANY SIZE by reference to a search string.
+  @details Provide a fileref and a search string to chop off part of a file.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the chop.
+
+  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
+  use an OFFSET to fix the precise chop point.
+
+  Usage:
+
+      %let src="%sysfunc(pathname(work))/file.txt";
+      %let str=Chop here!;
+      %let out1="%sysfunc(pathname(work))/file1.txt";
+      %let out2="%sysfunc(pathname(work))/file2.txt";
+      %let out3="%sysfunc(pathname(work))/file3.txt";
+      %let out4="%sysfunc(pathname(work))/file4.txt";
+
+      data _null_;
+        file &src;
+        put "startsection&str.endsection";
+      run;
+
+      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
+      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
+      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
+      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
+
+      filename results (&out1 &out2 &out3 &out4);
+      data _null_;
+        infile results;
+        input;
+        list;
+      run;
+
+  Results:
+    @li `startsection`
+    @li `Chop here!endsection`
+    @li `startsectionChop here!`
+    @li `endsection`
+
+  For more examples, see mp_chop.test.sas
+
+  @param [in] infile The QUOTED path to the file on which to perform the chop
+  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchpoint= (START) Valid values:
+    @li START - chop at the beginning of the string in `matchvar`.
+    @li END - chop at the end of the string in `matchvar`.
+  @param [in] offset= (0) An adjustment to the precise chop location, by
+    by reference to the `matchpoint`. Should be a positive or negative integer.
+  @param [in] keep= (FIRST) Valid values:
+    @li FIRST - keep the section of the file before the chop
+    @li LAST - keep the section of the file after the chop
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+    overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.sas
+  @li mp_chop.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_chop(infile,
+  matchvar=,
+  matchpoint=START,
+  keep=FIRST,
+  offset=0,
+  mdebug=0,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local fref0 dttm ds1 outref;
+%let fref0=%mf_getuniquefileref();
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+%if &outfile=0 %then %let outfile=&infile;
+
+%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
+  ,mac=mp_chop.sas
+  ,msg=%str(&matchvar is an empty variable)
+)
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &fref0 &infile lrecl=1 recfm=n;
+
+/* create dataset with one char per row */
+data &ds1;
+  infile &fref0;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+/* get start & stop position of first matchvar string (one row, two vars) */
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&matchvar));
+  string =symget("&matchvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+        /* matched! */
+        stop;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+%local split;
+%let split=0;
+data _null_;
+  set &ds2;
+  if "&matchpoint"='START' then do;
+    if "&keep"='FIRST' then mp=start;
+    else if "&keep"='LAST' then mp=start-1;
+  end;
+  else if "&matchpoint"='END' then do;
+    if "&keep"='FIRST' then mp=stop+1;
+    else if "&keep"='LAST' then mp=stop;
+  end;
+  split=mp+&offset;
+  call symputx('split',split,'l');
+%if &mdebug=1 %then %do;
+  put (_all_)(=);
+  %put &=offset;
+%end;
+run;
+%if &split=0 %then %do;
+  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
+  %return;
+%end;
+
+data _null_;
+  file &outfile recfm=n;
+  set &ds1;
+%if &keep=FIRST %then %do;
+  if _n_ ge &split then stop;
+%end;
+%else %do;
+  if _n_ gt &split;
+%end;
+  put sourcechar char1.;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &outfile lrecl=32767;
+    input;
+    list;
+    if _n_>50 then stop;
+  run;
+%end;
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_chop;