@sasjs/utils 2.48.2 → 2.49.1

package/macros/mp_binarycopy.sas

@@ -0,0 +1,79 @@
+/**
+  @file
+  @brief Copy any file using binary input / output streams
+  @details Reads in a file byte by byte and writes it back out.  Is an
+  os-independent method to copy files.  In case of naming collision, the
+  default filerefs can be modified.
+  Note that if you have a new enough version of SAS, and you don't need features
+  such as APPEND, you may be better of using the fcopy() function instead.
+
+        %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
+
+  To append to a file, use the mode option, eg:
+
+      filename tmp1 temp;
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
+
+  @returns nothing
+
+  @version 9.2
+
+**/
+
+%macro mp_binarycopy(
+    inloc=           /* full path and filename of the object to be copied */
+    ,outloc=          /* full path and filename of object to be created */
+    ,inref=____in   /* override default to use own filerefs */
+    ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
+    ,iftrue=%str(1=1)
+)/*/STORE SOURCE*/;
+  %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
+  %if &mode=APPEND %then %let mod=mod;
+
+  /* these IN and OUT filerefs can point to anything */
+  %if &inref = ____in %then %do;
+    filename &inref &inloc lrecl=1048576 ;
+  %end;
+  %if &outref=____out %then %do;
+    filename &outref &outloc lrecl=1048576 &mod;
+  %end;
+
+  /* copy the file byte-for-byte  */
+  data _null_;
+    infile &inref lrecl=1 recfm=n;
+    file &outref &mod recfm=n;
+    input sourcechar $char1. @@;
+    format sourcechar hex2.;
+    put sourcechar char1. @@;
+  run;
+
+  %if &inref = ____in %then %do;
+    filename &inref clear;
+  %end;
+  %if &outref=____out %then %do;
+    filename &outref clear;
+  %end;
+%mend mp_binarycopy;

package/macros/mp_chop.sas

@@ -0,0 +1,194 @@
+/**
+  @file
+  @brief Splits a file of ANY SIZE by reference to a search string.
+  @details Provide a fileref and a search string to chop off part of a file.
+
+  Works by reading in the file byte by byte, then marking the beginning and end
+  of each matched string, before finally doing the chop.
+
+  Choose whether to keep the FIRST or the LAST section of the file.  Optionally,
+  use an OFFSET to fix the precise chop point.
+
+  Usage:
+
+      %let src="%sysfunc(pathname(work))/file.txt";
+      %let str=Chop here!;
+      %let out1="%sysfunc(pathname(work))/file1.txt";
+      %let out2="%sysfunc(pathname(work))/file2.txt";
+      %let out3="%sysfunc(pathname(work))/file3.txt";
+      %let out4="%sysfunc(pathname(work))/file4.txt";
+
+      data _null_;
+        file &src;
+        put "startsection&str.endsection";
+      run;
+
+      %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1)
+      %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2)
+      %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3)
+      %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4)
+
+      filename results (&out1 &out2 &out3 &out4);
+      data _null_;
+        infile results;
+        input;
+        list;
+      run;
+
+  Results:
+    @li `startsection`
+    @li `Chop here!endsection`
+    @li `startsectionChop here!`
+    @li `endsection`
+
+  For more examples, see mp_chop.test.sas
+
+  @param [in] infile The QUOTED path to the file on which to perform the chop
+  @param [in] matchvar= Macro variable NAME containing the string to split by
+  @param [in] matchpoint= (START) Valid values:
+    @li START - chop at the beginning of the string in `matchvar`.
+    @li END - chop at the end of the string in `matchvar`.
+  @param [in] offset= (0) An adjustment to the precise chop location, by
+    by reference to the `matchpoint`. Should be a positive or negative integer.
+  @param [in] keep= (FIRST) Valid values:
+    @li FIRST - keep the section of the file before the chop
+    @li LAST - keep the section of the file after the chop
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param outfile= (0) Optional QUOTED path to the adjusted output file (avoids
+    overwriting the first file).
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_abort.sas
+  @li mp_gsubfile.sas
+  @li mp_replace.sas
+  @li mp_chop.test.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_chop(infile,
+  matchvar=,
+  matchpoint=START,
+  keep=FIRST,
+  offset=0,
+  mdebug=0,
+  outfile=0
+)/*/STORE SOURCE*/;
+
+%local fref0 dttm ds1 outref;
+%let fref0=%mf_getuniquefileref();
+%let ds1=%mf_getuniquename(prefix=allchars);
+%let ds2=%mf_getuniquename(prefix=startmark);
+
+%if &outfile=0 %then %let outfile=&infile;
+
+%mp_abort(iftrue= (%length(%superq(&matchvar))=0)
+  ,mac=mp_chop.sas
+  ,msg=%str(&matchvar is an empty variable)
+)
+
+/* START */
+%let dttm=%sysfunc(datetime());
+
+filename &fref0 &infile lrecl=1 recfm=n;
+
+/* create dataset with one char per row */
+data &ds1;
+  infile &fref0;
+  input sourcechar $char1. @@;
+  format sourcechar hex2.;
+run;
+
+/* get start & stop position of first matchvar string (one row, two vars) */
+data &ds2;
+  /* set find string to length in bytes to cover trailing spaces */
+  length string $ %length(%superq(&matchvar));
+  string =symget("&matchvar");
+  drop string;
+
+  firstchar=char(string,1);
+  findlen=lengthm(string); /* <- for trailing bytes */
+
+  do _N_=1 to nobs;
+    set &ds1 nobs=nobs point=_N_;
+    if sourcechar=firstchar then do;
+      pos=1;
+      s=0;
+      do point=_N_ to min(_N_ + findlen -1,nobs);
+        set &ds1 point=point;
+        if sourcechar=char(string, pos) then s + 1;
+        else goto _leave_;
+        pos+1;
+      end;
+      _leave_:
+      if s=findlen then do;
+        START =_N_;
+        _N_ =_N_+ s - 1;
+        STOP =_N_;
+        output;
+        /* matched! */
+        stop;
+      end;
+    end;
+  end;
+  stop;
+  keep START STOP;
+run;
+
+%local split;
+%let split=0;
+data _null_;
+  set &ds2;
+  if "&matchpoint"='START' then do;
+    if "&keep"='FIRST' then mp=start;
+    else if "&keep"='LAST' then mp=start-1;
+  end;
+  else if "&matchpoint"='END' then do;
+    if "&keep"='FIRST' then mp=stop+1;
+    else if "&keep"='LAST' then mp=stop;
+  end;
+  split=mp+&offset;
+  call symputx('split',split,'l');
+%if &mdebug=1 %then %do;
+  put (_all_)(=);
+  %put &=offset;
+%end;
+run;
+%if &split=0 %then %do;
+  %put &sysmacroname: No match found in &infile for string %superq(&matchvar);
+  %return;
+%end;
+
+data _null_;
+  file &outfile recfm=n;
+  set &ds1;
+%if &keep=FIRST %then %do;
+  if _n_ ge &split then stop;
+%end;
+%else %do;
+  if _n_ gt &split;
+%end;
+  put sourcechar char1.;
+run;
+
+%if &mdebug=0 %then %do;
+  filename &fref0 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &outfile lrecl=32767;
+    input;
+    list;
+    if _n_>50 then stop;
+  run;
+%end;
+/* END */
+%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run;
+
+%mend mp_chop;

package/macros/mp_cleancsv.sas

@@ -0,0 +1,78 @@
+/**
+  @file
+  @brief Fixes embedded cr / lf / crlf in CSV
+  @details CSVs will sometimes contain lf or crlf within quotes (eg when
+  saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
+  to process using SAS defaults.
+  This macro converts any csv to follow the convention of a windows excel file,
+  applying CRLF line endings and converting embedded cr and crlf to lf.
+
+  Usage:
+
+      fileref mycsv "/path/your/csv";
+      %mp_cleancsv(in=mycsv,out=/path/new.csv)
+
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
+%if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do;
+  %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations;
+  %return;
+%end;
+
+/* presence of a period(.) indicates a physical location */
+%if %index(&in,.) %then %let in="&in";
+%if %index(&out,.) %then %let out="&out";
+
+/**
+  * convert all cr and crlf within quotes to lf
+  * convert all other cr or lf to crlf
+  */
+  data _null_;
+    infile &in recfm=n ;
+    file &out recfm=n;
+    retain isq iscrlf 0 qchar &qchar;
+    input inchar $char1. ;
+    if inchar=qchar then isq = mod(isq+1,2);
+    if isq then do;
+      /* inside a quote change cr and crlf to lf */
+      if inchar='0D'x then do;
+        put '0A'x;
+        input inchar $char1.;
+        if inchar ne '0A'x then do;
+          put inchar $char1.;
+          if inchar=qchar then isq = mod(isq+1,2);
+        end;
+      end;
+      else put inchar $char1.;
+    end;
+    else do;
+      /* outside a quote, change cr and lf to crlf */
+      if inchar='0D'x then do;
+        crblank:
+        put '0D0A'x;
+        input inchar $char1.;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
+          put inchar $char1.;
+          if inchar=qchar then isq = mod(isq+1,2);
+        end;
+      end;
+      else if inchar='0A'x then put '0D0A'x;
+      else put inchar $char1.;
+    end;
+  run;
+%mend mp_cleancsv;
+/** @endcond */

package/macros/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

package/macros/mp_copyfolder.sas

@@ -0,0 +1,84 @@
+/**
+  @file
+  @brief A macro to recursively copy a directory
+  @details Performs a recursive directory listing then works from top to bottom
+    copying files and creating subdirectories.
+
+  Usage:
+
+      %let rootdir=%sysfunc(pathname(work))/demo;
+      %let copydir=%sysfunc(pathname(work))/demo_copy;
+      %mf_mkdir(&rootdir)
+      %mf_mkdir(&rootdir/subdir)
+      %mf_mkdir(&rootdir/subdir/subsubdir)
+      data "&rootdir/subdir/example.sas7bdat";
+      run;
+
+      %mp_copyfolder(&rootdir,&copydir)
+
+  @param [in] source Unquoted path to the folder to copy from.
+  @param [out] target Unquoted path to the folder to copy to.
+  @param [in] copymax=(MAX) Set to a positive integer to indicate the level of
+    subdirectory copy recursion - eg 3, to go `./3/levels/deep`.  For unlimited
+    recursion, set to MAX.
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mf_isdir.sas
+  @li mf_mkdir.sas
+  @li mp_abort.sas
+  @li mp_dirlist.sas
+
+  <h4> Related Macros </h4>
+  @li mp_copyfolder.test.sas
+
+**/
+
+%macro mp_copyfolder(source,target,copymax=MAX);
+
+  %mp_abort(iftrue=(%mf_isdir(&source)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Source dir does not exist (&source))
+  )
+
+  %mf_mkdir(&target)
+
+  %mp_abort(iftrue=(%mf_isdir(&target)=0)
+    ,mac=&sysmacroname
+    ,msg=%str(Target dir could not be created (&target))
+  )
+
+  /* prep temp table */
+  %local tempds;
+  %let tempds=%mf_getuniquename();
+
+  /* recursive directory listing */
+  %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=&copymax)
+
+  /* create folders and copy content */
+  data _null_;
+    length msg $200;
+    call missing(msg);
+    set work.&tempds;
+    if _n_ = 1 then dpos+sum(length(directory),2);
+    filepath2="&target/"!!substr(filepath,dpos);
+    if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')');
+    else do;
+      length fref1 fref2 $8;
+      rc1=filename(fref1,filepath,'disk','recfm=n');
+      rc2=filename(fref2,filepath2,'disk','recfm=n');
+      if fcopy(fref1,fref2) ne 0 then do;
+        msg=sysmsg();
+        putlog "%str(ERR)OR: Unable to copy " filepath " to " filepath2;
+        putlog msg=;
+      end;
+    end;
+    rc=filename(fref1);
+    rc=filename(fref2);
+  run;
+
+  /* tidy up */
+  proc sql;
+  drop table work.&tempds;
+
+%mend mp_copyfolder;

package/macros/mp_coretable.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Create the permanent Core tables
+  @details Several macros in the [core](https://github.com/sasjs/core) library
+    make use of permanent tables.  To avoid duplication in definitions, this
+    macro provides a central location for managing the corresponding DDL.
+
+  Note - this macro is likely to be deprecated in future in favour of a
+  dedicated "datamodel" folder (prefix mddl)
+
+  Any corresponding data would go in a seperate repo, to avoid this one
+  ballooning in size!
+
+  Example usage:
+
+      %mp_coretable(LOCKTABLE,libds=work.locktable)
+
+  @param [in] table_ref The type of table to create.  Example values:
+    @li DIFFTABLE
+    @li FILTER_DETAIL
+    @li FILTER_SUMMARY
+    @li LOCKANYTABLE
+    @li MAXKEYTABLE
+  @param [in] libds= (0) The library.dataset reference used to create the table.
+    If not provided, then the DDL is simply printed to the log.
+
+  <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_filterdetail.sas
+  @li mddl_dc_filtersummary.sas
+  @li mddl_dc_locktable.sas
+  @li mddl_dc_maxkeytable.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mp_filterstore.sas
+  @li mp_lockanytable.sas
+  @li mp_retainedkey.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_coretable(table_ref,libds=0
+)/*/STORE SOURCE*/;
+%local outds ;
+%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds));
+proc sql;
+%if &table_ref=DIFFTABLE %then %do;
+  %mddl_dc_difftable(libds=&outds)
+%end;
+%else %if &table_ref=LOCKTABLE %then %do;
+  %mddl_dc_locktable(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_SUMMARY %then %do;
+  %mddl_dc_filtersummary(libds=&outds)
+%end;
+%else %if &table_ref=FILTER_DETAIL %then %do;
+  %mddl_dc_filterdetail(libds=&outds)
+%end;
+%else %if &table_ref=MAXKEYTABLE %then %do;
+  %mddl_dc_maxkeytable(libds=&outds)
+%end;
+
+%if &libds=0 %then %do;
+  proc sql;
+  describe table &syslast;
+  drop table &syslast;
+%end;
+%mend mp_coretable;

package/macros/mp_createconstraints.sas

@@ -0,0 +1,70 @@
+/**
+  @file mp_createconstraints.sas
+  @brief Creates constraints
+  @details Takes the output from mp_getconstraints.sas as input
+
+        proc sql;
+        create table work.example(
+          TX_FROM float format=datetime19.,
+          DD_TYPE char(16),
+          DD_SOURCE char(2048),
+          DD_SHORTDESC char(256),
+          constraint pk primary key(tx_from, dd_type,dd_source),
+          constraint unq unique(tx_from, dd_type),
+          constraint nnn not null(DD_SHORTDESC)
+        );
+
+      %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+      %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES)
+      %mp_createconstraints(inds=work.constraints,outds=created,execute=YES)
+
+  @param inds= (work.mp_getconstraints) The input table containing the
+    constraint info
+  @param outds= (work.mp_createconstraints) A table containing the create
+    statements (create_statement column)
+  @param execute= (NO) To actually create, use YES.
+
+  <h4> Related Files </h4>
+  @li mp_getconstraints.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_createconstraints(inds=mp_getconstraints
+  ,outds=work.mp_createconstraints
+  ,execute=NO
+)/*/STORE SOURCE*/;
+
+proc sort data=&inds out=&outds;
+  by libref table_name constraint_name;
+run;
+
+data &outds;
+  set &outds;
+  by libref table_name constraint_name;
+  length create_statement $500;
+  if _n_=1 and "&execute"="YES" then call execute('proc sql;');
+  if first.constraint_name then do;
+    if constraint_type='PRIMARY' then type='PRIMARY KEY';
+    else type=constraint_type;
+    create_statement=catx(" ","alter table",libref,".",table_name
+      ,"add constraint",constraint_name,type,"(");
+    if last.constraint_name then
+      create_statement=cats(create_statement,column_name,");");
+    else create_statement=cats(create_statement,column_name,",");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  else if last.constraint_name then do;
+    create_statement=cats(column_name,");");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  else do;
+    create_statement=cats(column_name,",");
+    if "&execute"="YES" then call execute(create_statement);
+  end;
+  output;
+run;
+
+%mend mp_createconstraints;

package/macros/mp_createwebservice.sas

@@ -0,0 +1,32 @@
+/**
+  @file mp_createwebservice.sas
+  @brief Create a web service in SAS 9, Viya or SASjs Server
+  @details This is actually a wrapper for mx_createwebservice.sas, remaining
+  for legacy purposes.  For new apps, use mx_createwebservice.sas.
+
+
+  <h4> SAS Macros </h4>
+  @li mx_createwebservice.sas
+
+
+**/
+
+%macro mp_createwebservice(path=HOME
+    ,name=initService
+    ,precode=
+    ,code=ft15f001
+    ,desc=This service was created by the mp_createwebservice macro
+    ,replace=YES
+    ,mdebug=0
+)/*/STORE SOURCE*/;
+
+  %mx_createwebservice(path=&path
+    ,name=&name
+    ,precode=&precode
+    ,code=&code
+    ,desc=&desc
+    ,replace=&replace
+    ,mdebug=&mdebug
+  )
+
+%mend mp_createwebservice;